diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..6992d1d306f6a01cd2a4ed4f13d9d3aa17a11762 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,34 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +37t5ymwr/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hcp6xxlt/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag filter=lfs diff=lfs merge=lfs -text diff --git a/231uxqco/.hydra/config.yaml b/231uxqco/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/231uxqco/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/231uxqco/.hydra/hydra.yaml b/231uxqco/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc5d2c02bde93d5bf0e0e1916e26036452999d3 --- /dev/null +++ b/231uxqco/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/231uxqco/.hydra/overrides.yaml b/231uxqco/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/231uxqco/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/231uxqco/episode_rosbags/aligned_depth_to_color_K.npy b/231uxqco/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/231uxqco/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/231uxqco/episode_rosbags/cam_tf_world.npy b/231uxqco/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/231uxqco/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/231uxqco/episode_rosbags/color_K.npy b/231uxqco/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/231uxqco/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/231uxqco/episode_rosbags/depth_K.npy b/231uxqco/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/231uxqco/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag b/231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..a7f9f22dd203c0b6cb9a25b180b37f3e7dcc7d4a --- /dev/null +++ b/231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573a290d1b5698e107277f3994c6725e53370a40a574533e8571fa813bacb694 +size 1059164155 diff --git a/231uxqco/eval_robot.log b/231uxqco/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6336448d5c735e7a9068f86431538df32c2035e5 --- /dev/null +++ b/231uxqco/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:02:55,810][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:02:55,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/231uxqco/eval_video/0_eval.mp4 b/231uxqco/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fc740fd33942e954a82c066eadcbbdd9b5f4f20 --- /dev/null +++ b/231uxqco/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79182f80255d44a832097c9a167b4ffae2e85c664d4cd1e225f3173ecfd26682 +size 732693 diff --git a/231uxqco/eval_video/1_eval.mp4 b/231uxqco/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05b2bd17c1594960240b05c1b7fdc5925808c1ca --- /dev/null +++ b/231uxqco/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd15d97a4d935b9d7c4771fad495abd63a13d1eea7cc7d360244e746fc26f6c9 +size 738914 diff --git a/231uxqco/eval_video/2_eval.mp4 b/231uxqco/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e531ce13f61354c6ac2c2d4b9c63946856004642 --- /dev/null +++ b/231uxqco/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a63220f48203a65003fc8aea34ee07e3dc74e5b4075a8732409ba655029a2c0 +size 727867 diff --git a/231uxqco/eval_video/3_eval.mp4 b/231uxqco/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f8f1ba2ec0e017cfeb8ff61c2ed71c63c8919a9 --- /dev/null +++ b/231uxqco/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a630738d7570f95eb6b4dd484151384f879e4686e4928bd530d2f1f845b2c458 +size 1498187 diff --git a/231uxqco/eval_video/4_eval.mp4 b/231uxqco/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ad82d0dd853fe04e290d29984eb7649ee47f8e3 --- /dev/null +++ b/231uxqco/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49fbacdcf9023c6df72d37d4a0147471cbbdb7c93388382fda2284b8dd1c0982 +size 867093 diff --git a/231uxqco/tb/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 b/231uxqco/tb/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 new file mode 100644 index 0000000000000000000000000000000000000000..9f54eff4fa9373169aec97823e2e1a89ea5daa92 --- /dev/null +++ b/231uxqco/tb/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9b62ce374a8c9b392cf88ef2232d5190594d9fd01b5ccea4be3984e754f96a +size 6499 diff --git a/231uxqco/wandb/debug-internal.log b/231uxqco/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ad2bc49d29c6ff2f62d1ab6f1671f25e9a0e4975 --- /dev/null +++ b/231uxqco/wandb/debug-internal.log @@ -0,0 +1,681 @@ +2025-01-21 21:03:01,629 INFO StreamThr :2644525 [internal.py:wandb_internal():85] W&B internal server running at pid: 2644525, started at: 2025-01-21 21:03:01.629061 +2025-01-21 21:03:01,630 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:03:01,638 INFO WriterThread:2644525 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb +2025-01-21 21:03:01,639 DEBUG SenderThread:2644525 [sender.py:send():379] send: header +2025-01-21 21:03:01,646 DEBUG SenderThread:2644525 [sender.py:send():379] send: run +2025-01-21 21:03:01,956 INFO SenderThread:2644525 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files +2025-01-21 21:03:01,956 INFO SenderThread:2644525 [sender.py:_start_run_threads():1188] run started: 231uxqco with start time 1737511381.629017 +2025-01-21 21:03:01,971 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:03:01,971 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:03:02,018 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:03:02,023 DEBUG HandlerThread:2644525 [system_info.py:__init__():26] System info init +2025-01-21 21:03:02,023 DEBUG HandlerThread:2644525 [system_info.py:__init__():41] System info init done +2025-01-21 21:03:02,023 INFO HandlerThread:2644525 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:03:02,023 INFO SystemMonitor:2644525 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:03:02,023 INFO HandlerThread:2644525 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:03:02,024 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:03:02,025 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:03:02,025 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:03:02,026 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:03:02,026 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:03:02,038 DEBUG HandlerThread:2644525 [system_info.py:probe():152] Probing system +2025-01-21 21:03:02,039 DEBUG HandlerThread:2644525 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:probe():200] Probing system done +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:03:02.038107', 'startedAt': '2025-01-22T02:03:01.622424', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3373.7529999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.064, 'min': 800.0, 'max': 4700.0}, {'current': 4500.015, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 337.3622856140137}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:03:02,044 INFO HandlerThread:2644525 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:03:02,044 INFO HandlerThread:2644525 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:_save_code():44] Saving code +2025-01-21 21:03:02,046 DEBUG HandlerThread:2644525 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:03:02,046 DEBUG HandlerThread:2644525 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:03:02,110 DEBUG HandlerThread:2644525 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:03:02,110 INFO HandlerThread:2644525 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:03:02,113 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:02,113 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:03:02,114 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:03:02,114 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:03:02,377 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:03:02,377 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:03:02,378 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/3zkzb5xu-wandb-metadata.json +2025-01-21 21:03:02,379 DEBUG SenderThread:2644525 [sender.py:send():379] send: telemetry +2025-01-21 21:03:02,396 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/8jizl6co-code/FISH/eval_robot.py +2025-01-21 21:03:02,398 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/11st59b2-diff.patch +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:02,421 INFO HandlerThread:2644525 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/tb" + save: true +} +control { + relay_id: "76f43b1f454548afa8c2604e17d6b8cd" +} +_info { + stream_id: "231uxqco" +} + +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:03:02,422 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:02,538 DEBUG SenderThread:2644525 [sender.py:send():379] send: tbrecord +2025-01-21 21:03:02,538 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:02,538 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 with policy live +2025-01-21 21:03:02,754 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/8rme17ih-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:02,957 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:02,957 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH +2025-01-21 21:03:03,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:04,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:04,956 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:05,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:06,303 DEBUG SenderThread:2644525 [sender.py:send():379] send: config +2025-01-21 21:03:06,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:07,311 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:07,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:08,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:08,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:09,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:10,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:10,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:11,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:12,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:12,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:12,959 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:13,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:14,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:14,959 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:15,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:16,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:16,960 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:17,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:17,378 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:17,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:17,484 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:18,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:18,961 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:19,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:20,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:20,961 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:21,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:22,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:23,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:23,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:24,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:25,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:26,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:27,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:28,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:28,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:29,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:30,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:31,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:32,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:32,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:32,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:33,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:33,543 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:33,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:03:34,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:34,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:35,319 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:35,319 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 with policy now +2025-01-21 21:03:35,319 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:03:35,320 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:03:35,320 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:03:35,320 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:03:35,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:35,537 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/4ma0xxq6-media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:35,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:03:35,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:03:36,249 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/zynl9cc8-media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:36,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:36,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:37,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:38,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:38,839 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:39,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:40,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:40,967 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:41,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:42,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:42,968 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:43,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:44,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:44,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:45,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:45,969 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:46,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:46,969 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:47,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:47,378 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:47,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:48,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:48,970 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:49,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:50,356 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:50,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:50,971 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:51,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:52,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:53,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:54,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:55,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:55,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:56,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:57,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:58,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:59,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:00,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:01,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:01,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:02,027 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:04:02,027 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:04:02,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:02,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:03,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:04,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:04,825 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:04:04,825 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 with policy now +2025-01-21 21:04:04,828 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:04:04,828 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:04,828 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:04,828 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:04:05,047 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/3d6pzxw4-media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:05,317 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/x01qzhh1-media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:05,387 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:06,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:06,848 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:06,976 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:07,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:08,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:09,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:10,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:10,977 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:11,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:12,121 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:12,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:12,863 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:12,863 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:12,864 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:13,238 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/2nefdmy5-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:13,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:14,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:14,979 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:15,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:16,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:16,979 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:17,273 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:17,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:17,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:17,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:18,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:18,980 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:19,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:20,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:21,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:22,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:22,429 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:23,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:24,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:24,982 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:25,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:26,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:27,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:27,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:28,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:29,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:30,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:31,400 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:31,457 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:04:31,458 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:04:31,458 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 with policy now +2025-01-21 21:04:31,458 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:31,459 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:31,459 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:31,674 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/pe370zgi-media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:31,984 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:31,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:31,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:04:32,028 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:04:32,262 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/2erluoyt-media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:32,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:32,385 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:32,397 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:32,529 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:32,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:33,399 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:34,399 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:35,400 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:36,402 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:36,986 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:37,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:37,689 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:38,401 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:38,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:39,401 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:39,596 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:39,596 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:39,597 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:39,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:39,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:40,404 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:40,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:41,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:42,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:42,988 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:43,211 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:43,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:44,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:44,989 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:45,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:46,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:47,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:47,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:47,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:48,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:48,539 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:49,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:50,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:51,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:52,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:53,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:54,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:54,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:55,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:56,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:57,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:58,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:59,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:59,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:00,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:01,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:02,028 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:05:02,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:02,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:03,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:04,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:04,437 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:05,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:06,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:07,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:08,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:08,997 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:09,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:10,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:10,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:11,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:12,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:13,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:14,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:15,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:16,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:16,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:17,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:17,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:17,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:18,411 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:19,000 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:19,032 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:05:19,032 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 with policy now +2025-01-21 21:05:19,036 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:05:19,036 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:19,036 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:19,061 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:19,263 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/bzt61uyu-media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:19,411 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:05:20,327 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/rsoxhdys-media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:20,412 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:21,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:21,414 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:22,062 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:22,416 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:23,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:24,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:25,002 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:25,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:26,420 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:27,003 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:27,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:27,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:28,418 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:29,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:30,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:31,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:32,029 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:05:32,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:32,383 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:32,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:32,489 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:32,659 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:32,659 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:32,659 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:33,005 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:05:33,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:33,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:33,275 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/tntmevn7-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:05:33,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:34,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:35,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:35,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:36,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:37,007 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:37,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:37,593 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:38,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:39,008 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:39,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:40,437 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:41,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:42,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:43,009 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:43,389 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:43,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:44,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:45,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:46,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:47,010 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:47,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:47,380 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:47,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:48,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:48,504 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:49,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:50,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:51,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:52,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:53,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:53,868 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:54,433 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:55,013 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:55,284 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:05:55,284 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 with policy now +2025-01-21 21:05:55,287 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:05:55,287 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:55,287 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:55,288 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:55,429 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:55,496 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/tbh4gbt4-media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:05:56,300 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/hwbhvjk4-media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,436 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:57,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:57,439 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:58,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:58,438 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:59,439 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:59,853 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:00,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:01,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:01,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:02,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:02,032 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:06:02,105 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:06:02,105 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:06:02,296 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:06:02,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:02,441 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:02,616 INFO wandb-upload_2:2644525 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk6rq4lnj +2025-01-21 21:06:02,648 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:06:02,658 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:06:02,992 INFO SenderThread:2644525 [sender.py:send_request_log_artifact():1525] logged artifact run-231uxqco-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0NDUyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MTQwNA==', 'latestArtifact': None}} +2025-01-21 21:06:02,992 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:06:02,993 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:02,993 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:03,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send():379] send: telemetry +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,131 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,132 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,132 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,132 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,132 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,133 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,133 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,133 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,133 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,134 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,134 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,134 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,134 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,135 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,135 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,135 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,135 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,136 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,136 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,136 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,136 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:06:03,187 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/sbhku7kw-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,213 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/m22i2phz-media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:03,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:03,454 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/6fkgp7r5-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,540 INFO wandb-upload_0:2644525 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd9y6kq6q +2025-01-21 21:06:04,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:04,080 INFO SenderThread:2644525 [sender.py:send_request_log_artifact():1525] logged artifact run-231uxqco-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0NDU1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MTQwNw==', 'latestArtifact': None}} +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:06:04,080 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json with policy now +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:04,081 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:04,081 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:04,081 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:04,085 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:04,362 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/adpbjchu-media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:04,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:05,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:05,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:05,085 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:05,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:06,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:07,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:08,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:09,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:10,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:10,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:11,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:12,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:13,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:14,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:15,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:16,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:16,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:17,022 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:06:17,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:17,380 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:17,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:18,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:19,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:20,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:21,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:22,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:22,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:23,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:24,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:25,162 DEBUG SenderThread:2644525 [sender.py:send():379] send: exit +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():588] handling runtime: 203 +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():594] send defer +2025-01-21 21:06:25,163 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,163 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:06:25,163 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,163 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:06:25,163 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:06:25,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:06:25,164 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,164 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:06:25,164 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:06:25,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:06:25,164 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:06:25,164 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:06:25,165 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:06:25,165 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:06:25,217 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,217 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:06:25,217 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,218 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:06:26,026 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:27,026 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:31,703 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:31,704 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:31,704 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,704 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:06:31,704 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:06:31,704 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:06:31,705 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:31,705 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,705 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:06:31,705 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:31,706 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:31,706 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:31,707 DEBUG SenderThread:2644525 [sender.py:send():379] send: summary +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:31,707 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:06:31,707 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,707 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:06:31,708 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:06:31,708 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:31,708 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,708 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:06:31,708 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:06:32,028 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:32,163 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:06:33,231 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:33,231 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:33,231 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:06:33,231 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:06:33,234 INFO SenderThread:2644525 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:06:33,236 INFO SenderThread:2644525 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:06:33,251 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:06:33,251 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:33,251 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:06:33,251 DEBUG SenderThread:2644525 [sender.py:send():379] send: artifact +2025-01-21 21:06:33,486 INFO SenderThread:2644525 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:06:33,487 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:33,487 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:06:33,487 INFO SenderThread:2644525 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml config.yaml +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json wandb-summary.json +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt requirements.txt +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log output.log +2025-01-21 21:06:34,032 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch diff.patch +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:06:34,039 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,039 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:06:34,039 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:06:34,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,164 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,223 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:34,237 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:06:34,261 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt +2025-01-21 21:06:34,341 INFO wandb-upload_4:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:34,349 INFO wandb-upload_3:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/qbsefj66-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:34,549 INFO Thread-11 (_thread_body):2644525 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:06:34,549 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,549 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:06:34,550 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,550 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:06:34,550 INFO SenderThread:2644525 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:06:34,551 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:06:34,551 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,551 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:06:34,552 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,552 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:06:34,552 INFO SenderThread:2644525 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:06:34,773 INFO SenderThread:2644525 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:06:34,773 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:06:34,774 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,774 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:06:34,774 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,774 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send():379] send: final +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send():379] send: footer +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,775 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,776 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:06:34,776 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:06:34,778 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:06:34,778 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:34,829 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,830 INFO MainThread:2644525 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:06:34,831 INFO MainThread:2644525 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:06:34,831 INFO MainThread:2644525 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:06:34,831 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:06:34,831 INFO HandlerThread:2644525 [handler.py:finish():882] shutting down handler +2025-01-21 21:06:35,776 INFO WriterThread:2644525 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [sender.py:finish():1615] shutting down sender +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [file_pusher.py:join():175] waiting for file pusher diff --git a/231uxqco/wandb/debug.log b/231uxqco/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f0c46bede0093b28fa7ef5c9893ca803509e04e8 --- /dev/null +++ b/231uxqco/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Configure stats pid to 2644351 +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/settings +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug.log +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug-internal.log +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco'} +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():619] starting backend +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():623] setting up manager +2025-01-21 21:03:01,628 INFO MainThread:2644351 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:03:01,628 INFO MainThread:2644351 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:03:01,638 INFO MainThread:2644351 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:03:01,644 INFO MainThread:2644351 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:03:01,970 INFO MainThread:2644351 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:03:02,013 INFO MainThread:2644351 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:03:02,014 INFO MainThread:2644351 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:03:02,380 INFO MainThread:2644351 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:03:02,380 INFO MainThread:2644351 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/tb, True +2025-01-21 21:03:06,303 INFO MainThread:2644351 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:06:35,832 WARNING MsgRouterThr:2644351 [router.py:message_loop():77] message_loop has been closed diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py b/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml b/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c163e881c56fa46f3dc0adea6e989b6f7ed1018 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737511381 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch b/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f977cc5199d71bcf6229d56fe4abffe631a81fa4 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[2, 15, 0.13333333333333333, 0.012688301601120637, 0.43005549909083207, 0.02801970029376167, 0.4017714858251573, 0.03871402898715848, 0.31777475777932684, 0.040400557311902036, 0.31017325822335207]]} \ No newline at end of file diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fc740fd33942e954a82c066eadcbbdd9b5f4f20 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79182f80255d44a832097c9a167b4ffae2e85c664d4cd1e225f3173ecfd26682 +size 732693 diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05b2bd17c1594960240b05c1b7fdc5925808c1ca --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd15d97a4d935b9d7c4771fad495abd63a13d1eea7cc7d360244e746fc26f6c9 +size 738914 diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e531ce13f61354c6ac2c2d4b9c63946856004642 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a63220f48203a65003fc8aea34ee07e3dc74e5b4075a8732409ba655029a2c0 +size 727867 diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1f8f1ba2ec0e017cfeb8ff61c2ed71c63c8919a9 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a630738d7570f95eb6b4dd484151384f879e4686e4928bd530d2f1f845b2c458 +size 1498187 diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ad82d0dd853fe04e290d29984eb7649ee47f8e3 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49fbacdcf9023c6df72d37d4a0147471cbbdb7c93388382fda2284b8dd1c0982 +size 867093 diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log b/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..807bc345a4ee0a4c936447ce450b865d851c6edd --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737511386.304271]: resetting environment +[INFO] [1737511386.308057]: cleared current plan +[INFO] [1737511386.308851]: moving to home +[INFO] [1737511387.311241]: reached home +[INFO] [1737511387.312964]: reset action history +[INFO] [1737511390.023274]: environment reset +Starting episode 0 +[INFO] [1737511390.027376]: resetting environment +[INFO] [1737511390.033133]: cleared current plan +[INFO] [1737511390.034608]: moving to home +[INFO] [1737511391.036807]: reached home +[INFO] [1737511391.038084]: reset action history +[INFO] [1737511393.798558]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737511395.934877]: resetting environment +[INFO] [1737511395.940018]: cleared current plan +[INFO] [1737511395.941919]: moving to home +[INFO] [1737511396.945487]: reached home +[INFO] [1737511396.947427]: reset action history +[INFO] [1737511399.655773]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/episode_rosbags/episode_0_2025-01-21-21-03-19.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2644351.log +started roslaunch server http://158.130.50.37:35401/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2644631] +process[print_text-2]: started with pid [2644655] +process[rosbag_record-3]: started with pid [2644656] +started bagging! +preempting episode +[INFO] [1737511414.282694]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737511415.077481]: Stored episode 1. +[INFO] [1737511415.078206]: Saving video... +[INFO] [1737511415.306006]: Video saved! +Starting episode 1 +[INFO] [1737511415.322939]: resetting environment +[INFO] [1737511415.336136]: cleared current plan +[INFO] [1737511415.336795]: moving to home +[INFO] [1737511418.839110]: reached home +[INFO] [1737511418.841480]: reset action history +[INFO] [1737511421.551880]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511425.394647]: resetting environment +[INFO] [1737511425.398777]: cleared current plan +[INFO] [1737511425.398938]: moving to home +[INFO] [1737511426.400367]: reached home +[INFO] [1737511426.400759]: reset action history +[INFO] [1737511429.107649]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/episode_rosbags/episode_1_2025-01-21-21-03-49.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2644351.log +started roslaunch server http://158.130.50.37:43413/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2644779] +process[print_text-5]: started with pid [2644780] +process[rosbag_record-6]: started with pid [2644804] +started bagging! +preempting episode +[INFO] [1737511443.733980]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1737511444.518746]: Stored episode 2. +[INFO] [1737511444.519584]: Saving video... +[INFO] [1737511444.805121]: Video saved! +Starting episode 2 +[INFO] [1737511444.832350]: resetting environment +[INFO] [1737511444.846761]: cleared current plan +[INFO] [1737511444.847655]: moving to home +[INFO] [1737511448.350158]: reached home +[INFO] [1737511448.355443]: reset action history +[INFO] [1737511451.076514]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511452.140847]: resetting environment +[INFO] [1737511452.150830]: cleared current plan +[INFO] [1737511452.151026]: moving to home +[INFO] [1737511453.152466]: reached home +[INFO] [1737511453.152965]: reset action history +[INFO] [1737511455.863679]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/episode_rosbags/episode_2_2025-01-21-21-04-15.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2644351.log +started roslaunch server http://158.130.50.37:42291/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2644952] +process[print_text-8]: started with pid [2644953] +process[rosbag_record-9]: started with pid [2644954] +started bagging! +For topic gripper_width: timestamp difference is 105999097 for nearest: 1737511462379171237 - target: 1737511462273172140 at idx 88 +preempting episode +[INFO] [1737511470.406128]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1737511471.121557]: Stored episode 3. +[INFO] [1737511471.124794]: Saving video... +[INFO] [1737511471.412588]: Video saved! +Starting episode 3 +[INFO] [1737511471.439085]: resetting environment +[INFO] [1737511471.454337]: cleared current plan +[INFO] [1737511471.455076]: moving to home +[INFO] [1737511474.960933]: reached home +[INFO] [1737511474.963919]: reset action history +[INFO] [1737511477.688791]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511479.454260]: resetting environment +[INFO] [1737511479.467877]: cleared current plan +[INFO] [1737511479.469798]: moving to home +[INFO] [1737511480.478248]: reached home +[INFO] [1737511480.481835]: reset action history +[INFO] [1737511483.210564]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/episode_rosbags/episode_3_2025-01-21-21-04-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2644351.log +started roslaunch server http://158.130.50.37:43829/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2645069] +process[print_text-11]: started with pid [2645070] +process[rosbag_record-12]: started with pid [2645094] +started bagging! +For topic gripper_width: timestamp difference is 109020337 for nearest: 1737511507279171332 - target: 1737511507170150995 at idx 249 +[INFO] [1737511517.900889]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1737511518.697542]: Stored episode 4. +[INFO] [1737511518.700881]: Saving video... +[INFO] [1737511518.963184]: Video saved! +Starting episode 4 +[INFO] [1737511519.044858]: resetting environment +[INFO] [1737511519.058009]: cleared current plan +[INFO] [1737511519.058241]: moving to home +[INFO] [1737511522.663255]: reached home +[INFO] [1737511522.667053]: reset action history +[INFO] [1737511525.455071]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511532.116097]: resetting environment +[INFO] [1737511532.142746]: cleared current plan +[INFO] [1737511532.143737]: moving to home +[INFO] [1737511533.149423]: reached home +[INFO] [1737511533.151736]: reset action history +[INFO] [1737511535.882608]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/episode_rosbags/episode_4_2025-01-21-21-05-35.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2644351.log +started roslaunch server http://158.130.50.37:35675/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2645245] +process[print_text-14]: started with pid [2645246] +process[rosbag_record-15]: started with pid [2645247] +started bagging! +For topic gripper_width: timestamp difference is 100174625 for nearest: 1737511537579171856 - target: 1737511537478997231 at idx 25 +For topic gripper_width: timestamp difference is 102285340 for nearest: 1737511540379172995 - target: 1737511540276887655 at idx 62 +For topic gripper_width: timestamp difference is 105865653 for nearest: 1737511544779203351 - target: 1737511544673337698 at idx 117 +For topic gripper_width: timestamp difference is 112680047 for nearest: 1737511553079175322 - target: 1737511552966495275 at idx 217 +preempting episode +[INFO] [1737511553.888417]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737511554.628613]: Stored episode 5. +[INFO] [1737511554.636236]: Saving video... +[INFO] [1737511555.240381]: Video saved! +[INFO] [1737511555.286868]: resetting environment +[INFO] [1737511555.296052]: cleared current plan +[INFO] [1737511555.296793]: moving to home +[WARN] [1737511555.188237]: Plan exhausted +[WARN] [1737511555.228219]: Plan exhausted +[WARN] [1737511555.268391]: Plan exhausted +[INFO] [1737511558.799912]: reached home +[INFO] [1737511558.807121]: reset action history +[INFO] [1737511561.595832]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt b/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json b/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ec05254d0a2a93079e283a408f3333bb253e1e24 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:03:02.038107", + "startedAt": "2025-01-22T02:03:01.622424", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3373.7529999999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.064, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.015, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 337.3622856140137 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json b/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..31fbb8bcacd9ae55b391e169434f0b974f2ef538 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "79182f80255d44a832097c9a167b4ffae2e85c664d4cd1e225f3173ecfd26682", "size": 732693, "path": "media/videos/eval/0_eval_0_79182f80255d44a83209.mp4"}, "global_step": 366, "_timestamp": 1737511562.5682266, "_runtime": 180.93920946121216, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "dd15d97a4d935b9d7c4771fad495abd63a13d1eea7cc7d360244e746fc26f6c9", "size": 738914, "path": "media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4"}, "eval/num_success": 1.0, "episode": 15.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "7a63220f48203a65003fc8aea34ee07e3dc74e5b4075a8732409ba655029a2c0", "size": 727867, "path": "media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a630738d7570f95eb6b4dd484151384f879e4686e4928bd530d2f1f845b2c458", "size": 1498187, "path": "media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "49fbacdcf9023c6df72d37d4a0147471cbbdb7c93388382fda2284b8dd1c0982", "size": 867093, "path": "media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://rdzm9nddnvykbmhi4brjdwi6nbyrgia47h9u8tqzzrulgr3tzn85c8ulks2htm6p9boqi6dsd8ef6z4nvjmg4u5d20qtz6j9ti61rj2y0by2puvt7kge9an0l9kkb2p3/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8yl9qgkq1pbmfmcrz9kg1lwihapit27aoimxhkwegh6q5n1jasy2whn0vex0uj1ol7j7t0y96qlticopan4hdwmdndzjba596oj2x4kq2nkav9ingzkri9ykjzavzkgo:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 2, "total_num_episodes": 15, "total_success_rate": 0.13333333333333333, "total_success_rate_lb_0.01": 0.012688301601120637, "total_success_rate_ub_0.01": 0.43005549909083207, "total_success_rate_lb_0.025": 0.02801970029376167, "total_success_rate_ub_0.025": 0.4017714858251573, "total_success_rate_lb_0.05": 0.03871402898715848, "total_success_rate_ub_0.05": 0.31777475777932684, "total_success_rate_lb_0.1": 0.040400557311902036, "total_success_rate_ub_0.1": 0.31017325822335207, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "ad9b7fb9cdbbec96b24f0132720174945a65d1e0650c2ff4bce15f6b1399f4ce", "size": 529, "artifact_path": "wandb-client-artifact://cceeh2usob9p7diob721d7ec4aqkexoza5e67hrofw9rn5gkdp1noqedi6qehk8x0q2ln46kob4l2yxfm3bdjc1xw87ujathevoqcb0k9wbsrxpq8g0adu7443f2i040/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://5q3g6eso0fx4o46dnpvm0r6jv24bwksrgkapguzdymudowohyj8lmhdf5zn541pge9au7da5wwbodm0ibl03hzsczxdm8kp491mvu7na7g2sv0kewehxb38zttjqtpi6:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 203}, "eval/total_success_rate": 0.13333334028720856, "eval/total_success_rate_lb_0.01": 0.012688301503658295, "eval/total_success_rate_ub_0.01": 0.43005549907684326, "eval/total_success_rate_lb_0.025": 0.028019700199365616, "eval/total_success_rate_ub_0.025": 0.4017714858055115, "eval/total_success_rate_lb_0.05": 0.038714028894901276, "eval/total_success_rate_ub_0.05": 0.31777477264404297, "eval/total_success_rate_lb_0.1": 0.04040055721998215, "eval/total_success_rate_ub_0.1": 0.31017327308654785} \ No newline at end of file diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug-internal.log b/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ad2bc49d29c6ff2f62d1ab6f1671f25e9a0e4975 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug-internal.log @@ -0,0 +1,681 @@ +2025-01-21 21:03:01,629 INFO StreamThr :2644525 [internal.py:wandb_internal():85] W&B internal server running at pid: 2644525, started at: 2025-01-21 21:03:01.629061 +2025-01-21 21:03:01,630 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:03:01,638 INFO WriterThread:2644525 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb +2025-01-21 21:03:01,639 DEBUG SenderThread:2644525 [sender.py:send():379] send: header +2025-01-21 21:03:01,646 DEBUG SenderThread:2644525 [sender.py:send():379] send: run +2025-01-21 21:03:01,956 INFO SenderThread:2644525 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files +2025-01-21 21:03:01,956 INFO SenderThread:2644525 [sender.py:_start_run_threads():1188] run started: 231uxqco with start time 1737511381.629017 +2025-01-21 21:03:01,971 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:03:01,971 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:03:02,018 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:03:02,023 DEBUG HandlerThread:2644525 [system_info.py:__init__():26] System info init +2025-01-21 21:03:02,023 DEBUG HandlerThread:2644525 [system_info.py:__init__():41] System info init done +2025-01-21 21:03:02,023 INFO HandlerThread:2644525 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:03:02,023 INFO SystemMonitor:2644525 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:03:02,023 INFO HandlerThread:2644525 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:03:02,024 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:03:02,025 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:03:02,025 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:03:02,026 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:03:02,026 INFO SystemMonitor:2644525 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:03:02,038 DEBUG HandlerThread:2644525 [system_info.py:probe():152] Probing system +2025-01-21 21:03:02,039 DEBUG HandlerThread:2644525 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:probe():200] Probing system done +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:03:02.038107', 'startedAt': '2025-01-22T02:03:01.622424', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3373.7529999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.064, 'min': 800.0, 'max': 4700.0}, {'current': 4500.015, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 337.3622856140137}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:03:02,044 INFO HandlerThread:2644525 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:03:02,044 INFO HandlerThread:2644525 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:03:02,044 DEBUG HandlerThread:2644525 [system_info.py:_save_code():44] Saving code +2025-01-21 21:03:02,046 DEBUG HandlerThread:2644525 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:03:02,046 DEBUG HandlerThread:2644525 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:03:02,110 DEBUG HandlerThread:2644525 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:03:02,110 INFO HandlerThread:2644525 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:03:02,113 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:02,113 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:03:02,114 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:03:02,114 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:03:02,377 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:03:02,377 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:03:02,378 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/3zkzb5xu-wandb-metadata.json +2025-01-21 21:03:02,379 DEBUG SenderThread:2644525 [sender.py:send():379] send: telemetry +2025-01-21 21:03:02,396 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/8jizl6co-code/FISH/eval_robot.py +2025-01-21 21:03:02,398 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/11st59b2-diff.patch +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:02,421 INFO HandlerThread:2644525 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/tb" + save: true +} +control { + relay_id: "76f43b1f454548afa8c2604e17d6b8cd" +} +_info { + stream_id: "231uxqco" +} + +2025-01-21 21:03:02,421 DEBUG HandlerThread:2644525 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:03:02,422 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:02,538 DEBUG SenderThread:2644525 [sender.py:send():379] send: tbrecord +2025-01-21 21:03:02,538 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:02,538 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 with policy live +2025-01-21 21:03:02,754 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/8rme17ih-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:02,957 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:02,957 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code +2025-01-21 21:03:02,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH +2025-01-21 21:03:03,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:04,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:04,956 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:05,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:06,303 DEBUG SenderThread:2644525 [sender.py:send():379] send: config +2025-01-21 21:03:06,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:07,311 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:07,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:08,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:08,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:09,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:10,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:10,958 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:11,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:12,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:12,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:12,959 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:13,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:14,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:14,959 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:15,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:16,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:16,960 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:17,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:17,378 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:17,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:17,484 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:18,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:18,961 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:19,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:20,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:20,961 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:21,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:22,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:23,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:23,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:24,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:25,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:26,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:27,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:28,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:28,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:29,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:30,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:31,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:32,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:32,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:32,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:33,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:33,543 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:33,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:03:34,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:34,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:35,319 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:03:35,319 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 with policy now +2025-01-21 21:03:35,319 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:03:35,320 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:03:35,320 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:03:35,320 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:03:35,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:35,537 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/4ma0xxq6-media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:35,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:03:35,965 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos +2025-01-21 21:03:35,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:03:36,249 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/zynl9cc8-media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:03:36,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:36,966 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:37,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:38,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:38,839 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:39,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:40,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:40,967 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:41,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:42,383 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:42,968 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:43,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:44,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:44,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:45,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:45,969 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:03:46,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:46,969 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:47,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:03:47,378 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:03:47,384 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:48,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:48,970 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:49,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:50,356 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:50,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:50,971 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:03:51,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:52,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:53,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:54,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:55,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:03:55,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:56,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:57,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:58,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:03:59,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:00,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:01,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:01,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:02,027 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:04:02,027 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:04:02,378 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:02,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:03,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:04,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:04,825 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:04:04,825 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 with policy now +2025-01-21 21:04:04,828 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:04:04,828 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:04,828 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:04,828 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:04,975 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:04:05,047 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/3d6pzxw4-media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:05,317 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/x01qzhh1-media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:04:05,387 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:06,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:06,848 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:06,976 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:07,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:08,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:09,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:10,396 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:10,977 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:11,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:12,121 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:12,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:12,863 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:12,863 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:12,864 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:12,978 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:13,238 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/2nefdmy5-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:13,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:14,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:14,979 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:15,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:16,392 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:16,979 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:17,273 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:17,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:17,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:17,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:18,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:18,980 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:19,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:20,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:21,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:22,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:22,429 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:23,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:24,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:24,982 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:25,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:26,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:27,381 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:27,393 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:28,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:29,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:30,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:31,400 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:31,457 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:04:31,458 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:04:31,458 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 with policy now +2025-01-21 21:04:31,458 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:31,459 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:31,459 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:31,674 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/pe370zgi-media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:31,984 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:31,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:31,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:04:32,028 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:04:32,262 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/2erluoyt-media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:04:32,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:32,385 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:32,397 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:32,529 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:32,985 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:33,399 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:34,399 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:35,400 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:36,402 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:36,986 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:37,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:37,689 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:38,401 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:38,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:39,401 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:39,596 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:04:39,596 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:04:39,597 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:04:39,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:04:39,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:04:40,404 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:40,987 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:41,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:42,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:42,988 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:43,211 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:43,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:44,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:44,989 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:04:45,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:46,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:47,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:04:47,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:04:47,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:48,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:48,539 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:49,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:50,406 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:51,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:52,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:53,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:54,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:54,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:55,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:56,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:57,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:58,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:04:59,385 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:04:59,407 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:00,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:01,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:02,028 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:05:02,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:02,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:02,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:03,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:04,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:04,437 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:05,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:06,408 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:07,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:08,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:08,997 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:09,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:10,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:10,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:11,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:12,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:13,409 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:14,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:15,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:16,386 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:16,410 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:17,379 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:17,379 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:17,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:18,411 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:19,000 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:19,032 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:05:19,032 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 with policy now +2025-01-21 21:05:19,036 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:05:19,036 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:19,036 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:19,061 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:19,263 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/bzt61uyu-media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:19,411 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:20,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:05:20,327 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/rsoxhdys-media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:05:20,412 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:21,001 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:21,414 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:22,062 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:22,416 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:23,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:24,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:25,002 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:25,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:26,420 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:27,003 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:27,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:27,419 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:28,418 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:29,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:30,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:31,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:32,029 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:05:32,382 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:32,383 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:32,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:32,489 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:32,659 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:32,659 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:32,659 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:33,005 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:05:33,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:33,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:33,275 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/tntmevn7-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:05:33,421 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:34,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:35,006 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:35,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:36,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:37,007 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:37,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:37,593 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:38,422 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:39,008 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:39,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:40,437 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:41,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:42,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:43,009 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:43,389 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:43,423 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:44,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:45,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:46,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:47,010 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:47,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:05:47,380 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:05:47,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:48,424 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:48,504 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:49,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:50,425 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:51,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:52,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:53,426 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:53,868 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:05:54,433 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:55,013 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:55,284 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:05:55,284 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 with policy now +2025-01-21 21:05:55,287 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:05:55,287 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:05:55,287 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:05:55,288 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:05:55,429 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:55,496 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/tbh4gbt4-media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval +2025-01-21 21:05:56,300 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/hwbhvjk4-media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:05:56,436 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:57,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:57,439 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:58,014 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:05:58,438 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:59,439 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:05:59,853 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:00,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:01,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:01,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:02,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:02,032 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:06:02,105 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:06:02,105 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:06:02,296 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:06:02,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:02,441 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:02,616 INFO wandb-upload_2:2644525 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk6rq4lnj +2025-01-21 21:06:02,648 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:06:02,658 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:06:02,992 INFO SenderThread:2644525 [sender.py:send_request_log_artifact():1525] logged artifact run-231uxqco-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0NDUyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MTQwNA==', 'latestArtifact': None}} +2025-01-21 21:06:02,992 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:06:02,993 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:02,993 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:02,993 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:03,016 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table +2025-01-21 21:06:03,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send():379] send: telemetry +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,131 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,131 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,132 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,132 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,132 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,132 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,133 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,133 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,133 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,133 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,134 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,134 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,134 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,134 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,135 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,135 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,135 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,135 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,136 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,136 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:03,136 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:03,136 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:06:03,187 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/sbhku7kw-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,213 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/m22i2phz-media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:03,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:03,454 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/6fkgp7r5-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:03,540 INFO wandb-upload_0:2644525 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd9y6kq6q +2025-01-21 21:06:04,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:04,080 INFO SenderThread:2644525 [sender.py:send_request_log_artifact():1525] logged artifact run-231uxqco-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0NDU1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MTQwNw==', 'latestArtifact': None}} +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send():379] send: files +2025-01-21 21:06:04,080 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json with policy now +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:04,080 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:04,081 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:04,081 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:04,081 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:04,085 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:04,362 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/adpbjchu-media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:04,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:05,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:05,017 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:05,085 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:05,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:06,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:07,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:08,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:09,442 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:10,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:10,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:11,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:12,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:13,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:14,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:15,443 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:16,394 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:16,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:17,022 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:06:17,380 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:17,380 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:17,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:18,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:19,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:20,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:21,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:22,391 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:22,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:23,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:24,444 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:25,162 DEBUG SenderThread:2644525 [sender.py:send():379] send: exit +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():588] handling runtime: 203 +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:25,162 INFO SenderThread:2644525 [sender.py:send_exit():594] send defer +2025-01-21 21:06:25,163 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,163 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:06:25,163 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,163 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:06:25,163 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:06:25,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:06:25,164 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,164 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:06:25,164 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:06:25,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:06:25,164 INFO HandlerThread:2644525 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:06:25,164 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:06:25,164 DEBUG SystemMonitor:2644525 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:06:25,165 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:06:25,165 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:06:25,216 INFO HandlerThread:2644525 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send():379] send: stats +2025-01-21 21:06:25,217 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,217 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:06:25,217 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:06:25,217 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:06:25,217 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:25,218 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:06:26,026 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:27,026 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:31,703 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:31,704 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:31,704 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,704 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:06:31,704 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:06:31,704 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:06:31,705 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:31,705 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,705 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:06:31,705 DEBUG SenderThread:2644525 [sender.py:send():379] send: history +2025-01-21 21:06:31,706 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:06:31,706 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:31,707 DEBUG SenderThread:2644525 [sender.py:send():379] send: summary +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:06:31,707 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:06:31,707 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:06:31,707 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,707 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:06:31,708 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:06:31,708 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:31,708 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:31,708 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:06:31,708 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:31,708 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:06:32,028 INFO Thread-12 :2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:32,163 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:06:33,231 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:33,231 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:33,231 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:06:33,231 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:06:33,231 INFO SenderThread:2644525 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:06:33,234 INFO SenderThread:2644525 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:06:33,236 INFO SenderThread:2644525 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:06:33,251 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:06:33,251 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:33,251 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:06:33,251 DEBUG SenderThread:2644525 [sender.py:send():379] send: artifact +2025-01-21 21:06:33,486 INFO SenderThread:2644525 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:06:33,487 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:33,487 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:06:33,487 INFO SenderThread:2644525 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml config.yaml +2025-01-21 21:06:34,029 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json wandb-summary.json +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt requirements.txt +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:34,031 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log output.log +2025-01-21 21:06:34,032 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/diff.patch diff.patch +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 media/videos/eval/0_eval_0_79182f80255d44a83209.mp4 +2025-01-21 21:06:34,035 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 media/videos/eval/2_eval_2_7a63220f48203a65003f.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 media/videos/eval/1_eval_1_dd15d97a4d935b9d7c47.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 media/videos/eval/4_eval_4_49fbacdcf9023c6df72d.mp4 +2025-01-21 21:06:34,036 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 media/videos/eval/3_eval_3_a630738d7570f95eb6b4.mp4 +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json media/table/eval/total_success_rate_ci_6_ad9b7fb9cdbbec96b24f.table.json +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:06:34,039 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,039 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:06:34,039 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:06:34,039 INFO SenderThread:2644525 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:06:34,164 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,164 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,223 INFO wandb-upload_2:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/wandb-summary.json +2025-01-21 21:06:34,237 INFO wandb-upload_0:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/config.yaml +2025-01-21 21:06:34,261 INFO wandb-upload_1:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/requirements.txt +2025-01-21 21:06:34,341 INFO wandb-upload_4:2644525 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/files/output.log +2025-01-21 21:06:34,349 INFO wandb-upload_3:2644525 [upload_job.py:push():130] Uploaded file /tmp/tmpe58d3s2cwandb/qbsefj66-events.out.tfevents.1737511382.leonmkim-ROG-Strix-G15CS-G15CS.2644351.0 +2025-01-21 21:06:34,549 INFO Thread-11 (_thread_body):2644525 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:06:34,549 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,549 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:06:34,550 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,550 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:06:34,550 INFO SenderThread:2644525 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:06:34,551 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:06:34,551 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,551 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:06:34,552 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,552 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:06:34,552 INFO SenderThread:2644525 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:06:34,773 INFO SenderThread:2644525 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:06:34,773 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:06:34,774 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,774 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:06:34,774 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:06:34,774 INFO HandlerThread:2644525 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send():379] send: final +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send():379] send: footer +2025-01-21 21:06:34,774 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: defer +2025-01-21 21:06:34,774 INFO SenderThread:2644525 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,775 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:06:34,775 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:06:34,776 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:06:34,776 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:06:34,778 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:06:34,778 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:34,829 DEBUG SenderThread:2644525 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:06:34,830 INFO MainThread:2644525 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:06:34,831 INFO MainThread:2644525 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:06:34,831 INFO MainThread:2644525 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:06:34,831 DEBUG HandlerThread:2644525 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:06:34,831 INFO HandlerThread:2644525 [handler.py:finish():882] shutting down handler +2025-01-21 21:06:35,776 INFO WriterThread:2644525 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [sender.py:finish():1615] shutting down sender +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:06:35,830 INFO SenderThread:2644525 [file_pusher.py:join():175] waiting for file pusher diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug.log b/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f0c46bede0093b28fa7ef5c9893ca803509e04e8 --- /dev/null +++ b/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Configure stats pid to 2644351 +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/settings +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug.log +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/wandb/run-20250121_210301-231uxqco/logs/debug-internal.log +2025-01-21 21:03:01,624 INFO MainThread:2644351 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco'} +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():619] starting backend +2025-01-21 21:03:01,625 INFO MainThread:2644351 [wandb_init.py:init():623] setting up manager +2025-01-21 21:03:01,628 INFO MainThread:2644351 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:03:01,628 INFO MainThread:2644351 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:03:01,638 INFO MainThread:2644351 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:03:01,644 INFO MainThread:2644351 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:03:01,970 INFO MainThread:2644351 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:03:02,013 INFO MainThread:2644351 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:03:02,014 INFO MainThread:2644351 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:03:02,378 INFO MainThread:2644351 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:03:02,380 INFO MainThread:2644351 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:03:02,380 INFO MainThread:2644351 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/231uxqco/tb, True +2025-01-21 21:03:06,303 INFO MainThread:2644351 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:06:35,832 WARNING MsgRouterThr:2644351 [router.py:message_loop():77] message_loop has been closed diff --git a/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb b/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb new file mode 100644 index 0000000000000000000000000000000000000000..661fa8d02f18a9874b2c249dc5a14ee423e760e0 Binary files /dev/null and b/231uxqco/wandb/run-20250121_210301-231uxqco/run-231uxqco.wandb differ diff --git a/37t5ymwr/.hydra/config.yaml b/37t5ymwr/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/37t5ymwr/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/37t5ymwr/.hydra/hydra.yaml b/37t5ymwr/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1d589f639ae6e457cee3c5307aa257a03ce6f80 --- /dev/null +++ b/37t5ymwr/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/37t5ymwr/.hydra/overrides.yaml b/37t5ymwr/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/37t5ymwr/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/37t5ymwr/episode_rosbags/aligned_depth_to_color_K.npy b/37t5ymwr/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/37t5ymwr/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/37t5ymwr/episode_rosbags/cam_tf_world.npy b/37t5ymwr/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/37t5ymwr/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/37t5ymwr/episode_rosbags/color_K.npy b/37t5ymwr/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/37t5ymwr/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/37t5ymwr/episode_rosbags/depth_K.npy b/37t5ymwr/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/37t5ymwr/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/37t5ymwr/eval_robot.log b/37t5ymwr/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..4e1fcf8ce79e17ecc9d92d76a11a7a0afadae9ed --- /dev/null +++ b/37t5ymwr/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 20:56:04,201][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:56:04,203][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/37t5ymwr/eval_video/0_eval.mp4 b/37t5ymwr/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d6c992fa3dec4c184598baa08524b3b624d9de0 --- /dev/null +++ b/37t5ymwr/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f683e3d27762b787fec968a43a002ce99ba89e75ad7489e6f4904d2285ad695 +size 1559689 diff --git a/37t5ymwr/eval_video/1_eval.mp4 b/37t5ymwr/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5fd0508ff040f8344d36961ea8778f0a3303749c --- /dev/null +++ b/37t5ymwr/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52f38aea11256dff566a2fe5eaa942024fa1dd1d934656e060ced805c2d27859 +size 967457 diff --git a/37t5ymwr/eval_video/2_eval.mp4 b/37t5ymwr/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25ca6c2598fe13517ffd5c9a35bee470aa3bec05 --- /dev/null +++ b/37t5ymwr/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b7b0943e5ffceb2ddf97a0139944e21ff99cec3cc94a868728bb81dbb119a6 +size 855954 diff --git a/37t5ymwr/eval_video/3_eval.mp4 b/37t5ymwr/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df4f56e6186d1ed12055f2e8493f41540ed9f4bd --- /dev/null +++ b/37t5ymwr/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1f7174555f6baf68df8b0ad0e48f5922021f91c0eaa6bdcf8fa1bbfde0c8f9 +size 1514034 diff --git a/37t5ymwr/eval_video/4_eval.mp4 b/37t5ymwr/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a845505d83b42cd2c03742e7c5cfde073a91f78 --- /dev/null +++ b/37t5ymwr/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88bcd3f25d83f27b5d9f0b70d22f2822bca00fd82b3655266eb8985d2231772 +size 798661 diff --git a/37t5ymwr/tb/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 b/37t5ymwr/tb/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 new file mode 100644 index 0000000000000000000000000000000000000000..b1939e22d295ed8817cfdcbc3178c977cd396a19 --- /dev/null +++ b/37t5ymwr/tb/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ebbbdf91d6e9a4aff3ca12d0790608dd5078513d6ed9e5f453c146eed2f596d +size 6519 diff --git a/37t5ymwr/wandb/debug-internal.log b/37t5ymwr/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3dd087dfcb88b4b63cb614598e605893df4b6a8 --- /dev/null +++ b/37t5ymwr/wandb/debug-internal.log @@ -0,0 +1,741 @@ +2025-01-21 20:56:09,991 INFO StreamThr :2643314 [internal.py:wandb_internal():85] W&B internal server running at pid: 2643314, started at: 2025-01-21 20:56:09.990696 +2025-01-21 20:56:09,992 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:56:10,004 INFO WriterThread:2643314 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb +2025-01-21 20:56:10,004 DEBUG SenderThread:2643314 [sender.py:send():379] send: header +2025-01-21 20:56:10,012 DEBUG SenderThread:2643314 [sender.py:send():379] send: run +2025-01-21 20:56:10,256 INFO SenderThread:2643314 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files +2025-01-21 20:56:10,256 INFO SenderThread:2643314 [sender.py:_start_run_threads():1188] run started: 37t5ymwr with start time 1737510969.993125 +2025-01-21 20:56:10,272 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:56:10,272 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:56:10,346 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:56:10,351 DEBUG HandlerThread:2643314 [system_info.py:__init__():26] System info init +2025-01-21 20:56:10,351 DEBUG HandlerThread:2643314 [system_info.py:__init__():41] System info init done +2025-01-21 20:56:10,351 INFO HandlerThread:2643314 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:56:10,351 INFO SystemMonitor:2643314 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:56:10,351 INFO HandlerThread:2643314 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:56:10,352 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:56:10,352 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:56:10,353 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:56:10,353 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:56:10,354 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:56:10,366 DEBUG HandlerThread:2643314 [system_info.py:probe():152] Probing system +2025-01-21 20:56:10,367 DEBUG HandlerThread:2643314 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_info.py:probe():200] Probing system done +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:56:10.366323', 'startedAt': '2025-01-22T01:56:09.983032', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.394875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.064, 'min': 800.0, 'max': 4700.0}, {'current': 4499.927, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4494.124, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.596, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 328.9591178894043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:56:10,373 INFO HandlerThread:2643314 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:56:10,373 INFO HandlerThread:2643314 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:56:10,373 DEBUG HandlerThread:2643314 [system_info.py:_save_code():44] Saving code +2025-01-21 20:56:10,374 DEBUG HandlerThread:2643314 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:56:10,374 DEBUG HandlerThread:2643314 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:56:10,437 DEBUG HandlerThread:2643314 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:56:10,438 INFO HandlerThread:2643314 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:56:10,441 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 20:56:10,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:56:10,687 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:56:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:10,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:10,705 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ymvw3hhs-wandb-metadata.json +2025-01-21 20:56:10,721 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/dxz3pcgn-diff.patch +2025-01-21 20:56:10,737 INFO HandlerThread:2643314 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/tb" + save: true +} +control { + relay_id: "ff0654662fbf43b68afe3b9bae7d9643" +} +_info { + stream_id: "37t5ymwr" +} + +2025-01-21 20:56:10,737 DEBUG HandlerThread:2643314 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:56:10,746 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ry0xbikx-code/FISH/eval_robot.py +2025-01-21 20:56:10,776 DEBUG SenderThread:2643314 [sender.py:send():379] send: telemetry +2025-01-21 20:56:10,788 DEBUG SenderThread:2643314 [sender.py:send():379] send: tbrecord +2025-01-21 20:56:10,790 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:56:10,791 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 with policy live +2025-01-21 20:56:11,009 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/z2442873-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code +2025-01-21 20:56:11,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:12,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:13,257 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:13,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:14,612 DEBUG SenderThread:2643314 [sender.py:send():379] send: config +2025-01-21 20:56:14,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:15,621 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:15,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:16,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:17,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:17,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:18,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:19,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:20,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:20,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:21,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:21,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:22,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:23,260 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:23,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:24,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:25,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:25,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:25,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:25,785 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:26,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:27,261 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:27,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:28,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:29,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:30,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:31,262 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:31,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:31,694 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:32,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:33,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:34,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:35,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:36,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:37,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:37,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:38,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:39,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:40,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:40,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:40,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:41,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:42,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:43,135 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:43,276 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 20:56:43,276 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:43,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:44,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:45,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:46,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:47,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:48,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:48,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:49,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:50,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:51,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:52,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:53,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:54,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:54,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:55,693 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:55,693 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:55,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:56,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:57,280 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:57,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:58,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:59,281 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:59,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:00,233 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:00,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:01,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:02,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:03,167 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:57:03,167 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 with policy now +2025-01-21 20:57:03,167 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:57:03,168 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:03,168 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:03,169 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:03,282 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:57:03,450 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ljz8atw8-media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:03,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:03,734 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/jjdfoqnu-media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:04,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:05,284 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:05,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:05,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:06,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:07,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:08,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:09,286 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:09,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:10,354 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:57:10,355 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:57:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:10,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:10,750 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:11,286 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:11,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:12,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:13,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:14,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:15,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:16,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:16,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:17,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:18,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:19,289 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:19,569 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/qoq3b4pq-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:19,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:20,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:21,290 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:21,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:21,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:22,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:23,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:24,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:25,291 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:25,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:25,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:25,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:26,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:26,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:27,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:28,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:29,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:30,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:31,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:32,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:32,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:33,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:34,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:35,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:36,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:37,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:37,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:38,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:39,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:40,355 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:57:40,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:40,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:40,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:41,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:42,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:42,857 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:43,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:44,460 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:57:44,460 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 with policy now +2025-01-21 20:57:44,460 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:57:44,461 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:44,461 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:44,461 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:44,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:44,732 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/5gdnijbn-media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,305 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:45,305 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:45,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:57:45,603 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/9j33oegm-media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:46,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:47,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:47,706 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:47,980 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:48,706 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:49,307 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:49,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:50,713 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:51,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:52,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:53,137 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:53,137 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:53,137 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:53,138 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:53,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:54,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:55,309 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:55,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:55,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:55,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:56,711 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:57,708 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:58,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:59,135 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:59,310 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:59,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:00,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:01,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:02,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:03,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:04,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:04,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:05,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:06,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:07,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:08,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:09,694 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:09,741 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:10,356 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:58:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:10,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:11,314 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:11,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:12,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:13,711 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:14,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:14,753 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:15,715 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:16,055 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:58:16,055 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 with policy now +2025-01-21 20:58:16,059 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:58:16,059 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:58:16,060 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:58:16,060 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:58:16,307 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/8bx9sbuu-media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:58:16,593 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/4exfffme-media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,714 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:17,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:17,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:18,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:19,317 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:19,716 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:20,433 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:20,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:21,317 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:21,717 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:22,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:23,318 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:23,716 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:24,319 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:58:24,592 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/7s31klt1-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:58:24,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:24,959 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:58:24,959 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:58:24,959 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:58:25,319 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:58:25,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:25,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:25,692 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:25,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:25,763 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:26,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:27,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:27,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:28,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:29,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:29,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:30,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:31,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:31,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:32,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:33,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:34,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:35,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:36,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:36,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:37,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:38,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:39,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:40,356 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:58:40,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:40,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:40,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:41,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:41,843 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:42,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:43,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:44,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:45,326 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:45,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:46,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:47,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:47,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:48,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:49,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:50,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:51,328 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:51,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:52,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:53,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:53,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:54,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:55,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:55,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:55,731 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:56,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:57,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:58,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:58,772 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:59,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:00,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:01,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:02,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:03,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:03,888 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:03,932 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:03,932 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 with policy now +2025-01-21 20:59:03,935 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:03,935 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:03,935 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:03,953 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:04,203 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/xcloc3yc-media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:59:04,667 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/6a9qdti2-media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,734 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:05,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:05,748 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:06,736 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:07,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:08,735 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:09,334 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:09,617 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:09,738 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:10,357 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:59:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:10,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:10,735 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:11,334 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:11,739 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:12,738 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:13,745 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:14,742 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:14,751 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:15,742 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:16,741 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:17,743 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:18,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:19,025 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:19,026 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:19,026 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:19,337 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:19,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:19,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:19,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:20,605 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:20,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:21,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:21,749 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:22,748 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:23,339 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:23,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:24,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:25,339 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:25,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:25,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:25,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:25,797 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:26,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:27,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:28,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:29,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:30,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:31,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:31,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:32,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:33,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:34,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:35,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:36,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:36,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:37,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:38,786 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:39,344 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:39,624 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:39,624 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 with policy now +2025-01-21 20:59:39,628 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:39,628 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:39,629 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:39,629 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:39,786 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:39,882 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/23e9vvo8-media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,344 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:40,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:59:40,366 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:59:40,647 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/v9dsyv5q-media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:40,700 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:40,788 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:41,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:41,795 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:41,808 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:42,799 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:43,793 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:44,799 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:45,346 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:45,796 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:46,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:47,061 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:47,061 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:47,061 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:47,061 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:47,086 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:59:47,086 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:59:47,238 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval +2025-01-21 20:59:47,545 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/qdrrd83r-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:47,556 INFO wandb-upload_1:2643314 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa2qxpetd +2025-01-21 20:59:47,616 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/edae6jwi-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:47,644 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:59:47,654 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:47,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:48,034 INFO SenderThread:2643314 [sender.py:send_request_log_artifact():1525] logged artifact run-37t5ymwr-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzNjk0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4OTg3NA==', 'latestArtifact': None}} +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:48,035 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,035 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,036 DEBUG SenderThread:2643314 [sender.py:send():379] send: telemetry +2025-01-21 20:59:48,036 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,039 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,040 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,040 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,041 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,041 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,042 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,042 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,042 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,043 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,043 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,044 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,044 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,044 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,044 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,045 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,045 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:59:48,299 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/codu6ha8-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:48,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval +2025-01-21 20:59:48,510 INFO wandb-upload_0:2643314 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdiqzwyf7 +2025-01-21 20:59:48,528 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/vcx0yrho-media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:48,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:48,926 INFO SenderThread:2643314 [sender.py:send_request_log_artifact():1525] logged artifact run-37t5ymwr-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzNjk2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4OTg4Mw==', 'latestArtifact': None}} +2025-01-21 20:59:48,926 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:48,926 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json with policy now +2025-01-21 20:59:48,927 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:48,927 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,927 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:49,238 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/4o86scus-media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:49,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:49,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:49,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:50,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:51,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:52,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:52,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:53,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:54,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:55,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:55,690 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:55,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:56,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:57,773 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:57,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:58,352 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 20:59:58,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:59,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:00,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:01,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:02,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:03,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:03,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:04,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:05,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:06,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:07,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:08,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:08,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:09,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:10,358 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 21:00:10,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:00:10,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:00:10,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:11,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:12,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:13,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:13,840 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:14,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:15,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:16,834 DEBUG SenderThread:2643314 [sender.py:send():379] send: exit +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():588] handling runtime: 246 +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():594] send defer +2025-01-21 21:00:16,834 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,834 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:00:16,835 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:00:16,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:00:16,835 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:00:16,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:00:16,835 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:00:16,835 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:00:16,836 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:00:16,836 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:00:16,889 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:00:16,890 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:00:16,890 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:00:16,890 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,890 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:00:16,890 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:00:16,890 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 21:00:16,891 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,891 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:00:16,891 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,891 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:00:16,891 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:00:16,891 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,891 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:00:17,359 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:19,360 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:23,067 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:23,067 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:23,068 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,069 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:00:23,069 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:00:23,069 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,069 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:00:23,071 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:23,071 DEBUG SenderThread:2643314 [sender.py:send():379] send: summary +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:23,075 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:00:23,075 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,075 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:00:23,075 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:00:23,076 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:00:23,076 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:23,076 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,076 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:00:23,076 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,076 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:00:23,362 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:23,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:26,491 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:00:26,491 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:26,492 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:26,492 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:00:26,492 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:26,492 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:00:26,492 INFO SenderThread:2643314 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:00:26,494 INFO SenderThread:2643314 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:00:26,496 INFO SenderThread:2643314 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:00:26,509 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:00:26,509 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:26,509 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:00:26,509 DEBUG SenderThread:2643314 [sender.py:send():379] send: artifact +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:00:26,770 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:00:26,836 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml config.yaml +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json wandb-summary.json +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt requirements.txt +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log output.log +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch diff.patch +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:00:27,367 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:27,367 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,367 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:00:27,373 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,373 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:00:27,373 INFO SenderThread:2643314 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:00:27,571 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 21:00:27,607 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:27,614 INFO wandb-upload_4:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:27,614 INFO wandb-upload_3:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt +2025-01-21 21:00:27,651 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/l08znhm2-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 21:00:27,836 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:27,836 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:27,851 INFO Thread-11 (_thread_body):2643314 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:00:27,851 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,851 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:00:27,851 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,852 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:00:27,852 INFO SenderThread:2643314 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:00:27,853 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,853 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:00:27,853 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:00:28,051 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:28,051 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:00:28,051 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:00:28,051 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:28,051 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send():379] send: final +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send():379] send: footer +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:28,052 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:00:28,052 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:28,053 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:00:28,053 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:00:28,056 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:28,100 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:28,100 INFO MainThread:2643314 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:00:28,101 INFO MainThread:2643314 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:00:28,101 INFO MainThread:2643314 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:00:28,101 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:00:28,101 INFO HandlerThread:2643314 [handler.py:finish():882] shutting down handler +2025-01-21 21:00:29,055 INFO WriterThread:2643314 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [sender.py:finish():1615] shutting down sender +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [file_pusher.py:join():175] waiting for file pusher diff --git a/37t5ymwr/wandb/debug.log b/37t5ymwr/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1148fa4ad184d770901ef26822b9ae008b6ab73d --- /dev/null +++ b/37t5ymwr/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Configure stats pid to 2643122 +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/settings +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug.log +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug-internal.log +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr'} +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():619] starting backend +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():623] setting up manager +2025-01-21 20:56:09,989 INFO MainThread:2643122 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:56:09,991 INFO MainThread:2643122 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:56:10,003 INFO MainThread:2643122 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:56:10,010 INFO MainThread:2643122 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:56:10,272 INFO MainThread:2643122 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:56:10,340 INFO MainThread:2643122 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:56:10,341 INFO MainThread:2643122 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:56:10,688 INFO MainThread:2643122 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:56:10,693 INFO MainThread:2643122 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:56:10,693 INFO MainThread:2643122 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/tb, True +2025-01-21 20:56:14,612 INFO MainThread:2643122 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:00:29,102 WARNING MsgRouterThr:2643122 [router.py:message_loop():77] message_loop has been closed diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0bb871a36a8dc7fd295e6365b120612a1c88fc1c --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737510969 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9c17f48cfaded8620a92cc08a7a37d8b055eb14a --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[1, 10, 0.1, 0.0015661475314179946, 0.4730094671303371, 0.007920116284557946, 0.441189557325681, 0.014195734732156313, 0.3262754977096869, 0.013229236104090908, 0.32676362994797725]]} \ No newline at end of file diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d6c992fa3dec4c184598baa08524b3b624d9de0 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f683e3d27762b787fec968a43a002ce99ba89e75ad7489e6f4904d2285ad695 +size 1559689 diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5fd0508ff040f8344d36961ea8778f0a3303749c --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52f38aea11256dff566a2fe5eaa942024fa1dd1d934656e060ced805c2d27859 +size 967457 diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25ca6c2598fe13517ffd5c9a35bee470aa3bec05 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b7b0943e5ffceb2ddf97a0139944e21ff99cec3cc94a868728bb81dbb119a6 +size 855954 diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df4f56e6186d1ed12055f2e8493f41540ed9f4bd --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1f7174555f6baf68df8b0ad0e48f5922021f91c0eaa6bdcf8fa1bbfde0c8f9 +size 1514034 diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a845505d83b42cd2c03742e7c5cfde073a91f78 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88bcd3f25d83f27b5d9f0b70d22f2822bca00fd82b3655266eb8985d2231772 +size 798661 diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..beb1c4cb10efda5fb0edbee67175912ba3ead68b --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737510974.613024]: resetting environment +[INFO] [1737510974.617350]: cleared current plan +[INFO] [1737510974.618128]: moving to home +[INFO] [1737510975.620844]: reached home +[INFO] [1737510975.622317]: reset action history +[INFO] [1737510978.335233]: environment reset +Starting episode 0 +[INFO] [1737510978.339118]: resetting environment +[INFO] [1737510978.344255]: cleared current plan +[INFO] [1737510978.345283]: moving to home +[INFO] [1737510979.347239]: reached home +[INFO] [1737510979.348469]: reset action history +[INFO] [1737510982.058042]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737510985.092473]: resetting environment +[INFO] [1737510985.100301]: cleared current plan +[INFO] [1737510985.101165]: moving to home +[INFO] [1737510986.104069]: reached home +[INFO] [1737510986.106149]: reset action history +[INFO] [1737510988.814970]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/episode_rosbags/episode_0_2025-01-21-20-56-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2643122.log +started roslaunch server http://158.130.50.37:44043/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2643431] +process[print_text-2]: started with pid [2643455] +process[rosbag_record-3]: started with pid [2643456] +started bagging! +For topic gripper_width: timestamp difference is 101434124 for nearest: 1737511002079203499 - target: 1737511001977769375 at idx 181 +Invalid key press, try again +For topic gripper_width: timestamp difference is 119509005 for nearest: 1737511018179209732 - target: 1737511018059700727 at idx 249 +preempting episode +[INFO] [1737511022.181797]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737511022.959733]: Stored episode 1. +[INFO] [1737511022.963250]: Saving video... +[INFO] [1737511023.156332]: Video saved! +Starting episode 1 +[INFO] [1737511023.169119]: resetting environment +[INFO] [1737511023.181119]: cleared current plan +[INFO] [1737511023.182109]: moving to home +[INFO] [1737511026.884779]: reached home +[INFO] [1737511026.887921]: reset action history +[INFO] [1737511029.648951]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511039.080228]: resetting environment +[INFO] [1737511039.084102]: cleared current plan +[INFO] [1737511039.084291]: moving to home +[INFO] [1737511040.085273]: reached home +[INFO] [1737511040.085841]: reset action history +[INFO] [1737511042.794490]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/episode_rosbags/episode_1_2025-01-21-20-57-22.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2643122.log +started roslaunch server http://158.130.50.37:38961/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2643590] +process[print_text-5]: started with pid [2643591] +process[rosbag_record-6]: started with pid [2643615] +started bagging! +For topic gripper_width: timestamp difference is 100738016 for nearest: 1737511043379175630 - target: 1737511043278437614 at idx 11 +preempting episode +[INFO] [1737511063.092128]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737511063.806331]: Stored episode 2. +[INFO] [1737511063.806582]: Saving video... +[WARN] [1737511064.335190]: Plan exhausted +[WARN] [1737511064.374097]: Plan exhausted +[WARN] [1737511064.414162]: Plan exhausted +[WARN] [1737511064.453975]: Plan exhausted +[INFO] [1737511064.435886]: Video saved! +Starting episode 2 +[INFO] [1737511064.464477]: resetting environment +[INFO] [1737511064.473880]: cleared current plan +[INFO] [1737511064.474143]: moving to home +[INFO] [1737511067.979635]: reached home +[INFO] [1737511067.981559]: reset action history +[INFO] [1737511070.754959]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511072.843319]: resetting environment +[INFO] [1737511072.853429]: cleared current plan +[INFO] [1737511072.853687]: moving to home +[INFO] [1737511073.855002]: reached home +[INFO] [1737511073.855884]: reset action history +[INFO] [1737511076.715407]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/episode_rosbags/episode_2_2025-01-21-20-57-56.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2643122.log +started roslaunch server http://158.130.50.37:46313/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2643756] +process[print_text-8]: started with pid [2643757] +process[rosbag_record-9]: started with pid [2643758] +started bagging! +For topic gripper_width: timestamp difference is -107321486 for nearest: 1737511089512532726 - target: 1737511089619854212 at idx 167 +preempting episode +[INFO] [1737511094.709667]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737511095.485336]: Stored episode 3. +[INFO] [1737511095.489623]: Saving video... +[INFO] [1737511096.019534]: Video saved! +Starting episode 3 +[INFO] [1737511096.062035]: resetting environment +[INFO] [1737511096.079572]: cleared current plan +[INFO] [1737511096.083352]: moving to home +[WARN] [1737511095.934242]: Plan exhausted +[WARN] [1737511095.974201]: Plan exhausted +[WARN] [1737511096.018020]: Plan exhausted +[WARN] [1737511096.055006]: Plan exhausted +[INFO] [1737511099.387600]: reached home +[INFO] [1737511099.393261]: reset action history +[INFO] [1737511102.126066]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511104.124163]: resetting environment +[INFO] [1737511104.146617]: cleared current plan +[INFO] [1737511104.151584]: moving to home +[INFO] [1737511105.155959]: reached home +[INFO] [1737511105.157106]: reset action history +[INFO] [1737511107.882132]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/episode_rosbags/episode_3_2025-01-21-20-58-27.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2643122.log +started roslaunch server http://158.130.50.37:33437/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2643906] +process[print_text-11]: started with pid [2643907] +process[rosbag_record-12]: started with pid [2643908] +started bagging! +For topic gripper_width: timestamp difference is -234596491 for nearest: 1737511123012508392 - target: 1737511123247104883 at idx 207 +For topic gripper_width: timestamp difference is 103258272 for nearest: 1737511128812507053 - target: 1737511128709248781 at idx 249 +[INFO] [1737511142.772923]: Storing episode... +[print_text-11] killing on exit +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1737511143.546363]: Stored episode 4. +[INFO] [1737511143.551099]: Saving video... +[INFO] [1737511143.886462]: Video saved! +Starting episode 4 +[INFO] [1737511143.937845]: resetting environment +[INFO] [1737511143.961232]: cleared current plan +[INFO] [1737511143.961670]: moving to home +[INFO] [1737511147.568050]: reached home +[INFO] [1737511147.576624]: reset action history +[INFO] [1737511150.308171]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511158.525803]: resetting environment +[INFO] [1737511158.549478]: cleared current plan +[INFO] [1737511158.552542]: moving to home +[INFO] [1737511159.559747]: reached home +[INFO] [1737511159.563035]: reset action history +[INFO] [1737511162.334472]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/episode_rosbags/episode_4_2025-01-21-20-59-22.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2643122.log +started roslaunch server http://158.130.50.37:41507/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2644058] +process[print_text-14]: started with pid [2644059] +process[rosbag_record-15]: started with pid [2644060] +started bagging! +preempting episode +[INFO] [1737511178.473168]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737511179.190926]: Stored episode 5. +[INFO] [1737511179.194999]: Saving video... +[INFO] [1737511179.581364]: Video saved! +[INFO] [1737511179.628774]: resetting environment +[INFO] [1737511179.651064]: cleared current plan +[INFO] [1737511179.655476]: moving to home +[INFO] [1737511183.167007]: reached home +[INFO] [1737511183.178329]: reset action history +[INFO] [1737511185.983485]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5e6bab6e2278ad05e7c0e1abcbd2ea7e3430125b --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T01:56:10.366323", + "startedAt": "2025-01-22T01:56:09.983032", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.394875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.064, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.927, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.124, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.596, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.005, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 328.9591178894043 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bab2ce6044eb517bfe4979ac4ebd106c6a47b8bb --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "2f683e3d27762b787fec968a43a002ce99ba89e75ad7489e6f4904d2285ad695", "size": 1559689, "path": "media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4"}, "global_step": 461, "_timestamp": 1737511187.5366125, "_runtime": 217.54348754882812, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "52f38aea11256dff566a2fe5eaa942024fa1dd1d934656e060ced805c2d27859", "size": 967457, "path": "media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4"}, "eval/num_success": 1.0, "episode": 10.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "02b7b0943e5ffceb2ddf97a0139944e21ff99cec3cc94a868728bb81dbb119a6", "size": 855954, "path": "media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "eb1f7174555f6baf68df8b0ad0e48f5922021f91c0eaa6bdcf8fa1bbfde0c8f9", "size": 1514034, "path": "media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e88bcd3f25d83f27b5d9f0b70d22f2822bca00fd82b3655266eb8985d2231772", "size": 798661, "path": "media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://jya6igk4418d9nfaxee3ec9l6vuohrgik9w6fkkm43lz405m3e8pmsc7uz6vmujkpu04q55ho5l8jco7q683c7tzbw858b6g3e4k1g3ua85k2g147uqbflkxjvdpn29l/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://47l03x2ddj6lqbnnqekwbbwj34a71jbxhtbkm7pw74cztntoyligaz4ikg0p8dto5pepqvhc2mcjzeevm1qn2b4en3lxnl160tpa5wgnzu40qfu7b2rrfqozqnwzaic7:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 1, "total_num_episodes": 10, "total_success_rate": 0.1, "total_success_rate_lb_0.01": 0.0015661475314179946, "total_success_rate_ub_0.01": 0.4730094671303371, "total_success_rate_lb_0.025": 0.007920116284557946, "total_success_rate_ub_0.025": 0.441189557325681, "total_success_rate_lb_0.05": 0.014195734732156313, "total_success_rate_ub_0.05": 0.3262754977096869, "total_success_rate_lb_0.1": 0.013229236104090908, "total_success_rate_ub_0.1": 0.32676362994797725, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "0c9cd2f25aeded733b20270186573d88a13c6ab3bb515cb04a4b297d88738d3e", "size": 513, "artifact_path": "wandb-client-artifact://7rkiucc3pn19mxu5bkcohrsxyit9umglgaehsycgcun21g3jqb8zwu98qg57t07dlveekxhut9iy055z4dz9xv1dgdwbwqdro9j0u0uft9cj6ypokgfxmk8wzpyg4kx9/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ud46t45efpn8cjl3kizuikw25gt75ww661ww3vr3692sm00il1n721jbwsdq4nx15cqssyuk9ws0dqqxn2yqy6hh58mbxqm3e5x0brkg83v1m308tvm5ut0ov9848dgy:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 246}, "eval/total_success_rate": 0.10000000149011612, "eval/total_success_rate_lb_0.01": 0.0015661475481465459, "eval/total_success_rate_ub_0.01": 0.47300946712493896, "eval/total_success_rate_lb_0.025": 0.007920116186141968, "eval/total_success_rate_ub_0.025": 0.44118955731391907, "eval/total_success_rate_lb_0.05": 0.01419573463499546, "eval/total_success_rate_ub_0.05": 0.326275497674942, "eval/total_success_rate_lb_0.1": 0.013229236006736755, "eval/total_success_rate_ub_0.1": 0.3267636299133301} \ No newline at end of file diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug-internal.log b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b3dd087dfcb88b4b63cb614598e605893df4b6a8 --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug-internal.log @@ -0,0 +1,741 @@ +2025-01-21 20:56:09,991 INFO StreamThr :2643314 [internal.py:wandb_internal():85] W&B internal server running at pid: 2643314, started at: 2025-01-21 20:56:09.990696 +2025-01-21 20:56:09,992 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:56:10,004 INFO WriterThread:2643314 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb +2025-01-21 20:56:10,004 DEBUG SenderThread:2643314 [sender.py:send():379] send: header +2025-01-21 20:56:10,012 DEBUG SenderThread:2643314 [sender.py:send():379] send: run +2025-01-21 20:56:10,256 INFO SenderThread:2643314 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files +2025-01-21 20:56:10,256 INFO SenderThread:2643314 [sender.py:_start_run_threads():1188] run started: 37t5ymwr with start time 1737510969.993125 +2025-01-21 20:56:10,272 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:56:10,272 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:56:10,346 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:56:10,351 DEBUG HandlerThread:2643314 [system_info.py:__init__():26] System info init +2025-01-21 20:56:10,351 DEBUG HandlerThread:2643314 [system_info.py:__init__():41] System info init done +2025-01-21 20:56:10,351 INFO HandlerThread:2643314 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:56:10,351 INFO SystemMonitor:2643314 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:56:10,351 INFO HandlerThread:2643314 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:56:10,352 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:56:10,352 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:56:10,353 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:56:10,353 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:56:10,354 INFO SystemMonitor:2643314 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:56:10,366 DEBUG HandlerThread:2643314 [system_info.py:probe():152] Probing system +2025-01-21 20:56:10,367 DEBUG HandlerThread:2643314 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_info.py:probe():200] Probing system done +2025-01-21 20:56:10,372 DEBUG HandlerThread:2643314 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:56:10.366323', 'startedAt': '2025-01-22T01:56:09.983032', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.394875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.064, 'min': 800.0, 'max': 4700.0}, {'current': 4499.927, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4494.124, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.596, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 328.9591178894043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:56:10,373 INFO HandlerThread:2643314 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:56:10,373 INFO HandlerThread:2643314 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:56:10,373 DEBUG HandlerThread:2643314 [system_info.py:_save_code():44] Saving code +2025-01-21 20:56:10,374 DEBUG HandlerThread:2643314 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:56:10,374 DEBUG HandlerThread:2643314 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:56:10,437 DEBUG HandlerThread:2643314 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:56:10,438 INFO HandlerThread:2643314 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:56:10,441 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:56:10,441 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 20:56:10,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:56:10,687 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:56:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:10,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:10,705 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ymvw3hhs-wandb-metadata.json +2025-01-21 20:56:10,721 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/dxz3pcgn-diff.patch +2025-01-21 20:56:10,737 INFO HandlerThread:2643314 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/tb" + save: true +} +control { + relay_id: "ff0654662fbf43b68afe3b9bae7d9643" +} +_info { + stream_id: "37t5ymwr" +} + +2025-01-21 20:56:10,737 DEBUG HandlerThread:2643314 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:56:10,746 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ry0xbikx-code/FISH/eval_robot.py +2025-01-21 20:56:10,776 DEBUG SenderThread:2643314 [sender.py:send():379] send: telemetry +2025-01-21 20:56:10,788 DEBUG SenderThread:2643314 [sender.py:send():379] send: tbrecord +2025-01-21 20:56:10,790 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:56:10,791 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 with policy live +2025-01-21 20:56:11,009 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/z2442873-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt +2025-01-21 20:56:11,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH +2025-01-21 20:56:11,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code +2025-01-21 20:56:11,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:12,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:13,257 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:13,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:14,612 DEBUG SenderThread:2643314 [sender.py:send():379] send: config +2025-01-21 20:56:14,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:15,621 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:15,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:16,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:17,258 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:17,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:18,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:19,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:20,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:20,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:21,259 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:21,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:22,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:23,260 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:23,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:24,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:25,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:25,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:25,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:25,785 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:26,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:27,261 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:27,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:28,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:29,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:30,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:31,262 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:31,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:31,694 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:32,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:33,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:34,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:35,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:36,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:37,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:37,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:38,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:39,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:40,687 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:40,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:40,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:41,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:42,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:43,135 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:43,276 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 20:56:43,276 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:43,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:44,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:45,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:46,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:47,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:48,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:48,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:49,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:50,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:51,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:52,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:53,697 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:54,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:56:54,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:55,693 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:56:55,693 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:56:55,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:56,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:57,280 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:57,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:58,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:56:59,281 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:56:59,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:00,233 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:00,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:01,698 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:02,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:03,167 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:57:03,167 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 with policy now +2025-01-21 20:57:03,167 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:57:03,168 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:03,168 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:03,169 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:03,282 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media +2025-01-21 20:57:03,283 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:57:03,450 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/ljz8atw8-media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:03,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:03,734 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/jjdfoqnu-media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 20:57:04,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:05,284 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:05,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:05,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:06,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:07,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:08,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:09,286 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:09,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:10,354 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:57:10,355 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:57:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:10,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:10,750 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:11,286 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:11,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:12,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:13,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:14,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:15,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:16,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:16,700 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:17,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:18,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:19,289 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:19,569 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/qoq3b4pq-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:19,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:20,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:21,290 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:21,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:21,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:22,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:23,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:24,701 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:25,291 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:25,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:25,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:25,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:26,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:26,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:27,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:28,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:29,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:30,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:31,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:32,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:32,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:33,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:34,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:35,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:36,703 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:37,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:37,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:38,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:39,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:40,355 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:57:40,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:40,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:40,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:41,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:42,704 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:42,857 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:43,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:44,460 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:57:44,460 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 with policy now +2025-01-21 20:57:44,460 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:57:44,461 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:44,461 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:44,461 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:44,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:44,732 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/5gdnijbn-media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,305 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:45,305 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:45,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:57:45,603 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/9j33oegm-media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 20:57:45,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:46,705 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:47,306 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:47,706 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:47,980 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:48,706 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:49,307 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:49,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:50,713 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:51,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:52,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:53,137 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:57:53,137 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:57:53,137 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:53,138 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:57:53,308 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:53,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:54,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:55,309 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:55,691 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:57:55,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:57:55,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:56,711 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:57,708 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:58,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:57:59,135 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:57:59,310 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:57:59,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:00,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:01,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:02,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:03,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:04,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:04,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:05,709 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:06,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:07,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:08,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:09,694 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:09,741 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:10,356 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:58:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:10,688 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:10,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:11,314 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:11,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:12,710 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:13,711 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:14,707 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:14,753 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:15,715 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:16,055 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:58:16,055 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 with policy now +2025-01-21 20:58:16,059 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:58:16,059 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:58:16,060 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:58:16,060 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:58:16,307 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/8bx9sbuu-media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:58:16,593 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/4exfffme-media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 20:58:16,714 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:17,316 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:17,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:18,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:19,317 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:19,716 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:20,433 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:20,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:21,317 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:21,717 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:22,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:23,318 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:23,716 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:24,319 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:58:24,592 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/7s31klt1-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:58:24,718 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:24,959 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:58:24,959 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:58:24,959 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:58:25,319 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:58:25,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:25,692 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:25,692 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:25,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:25,763 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:26,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:27,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:27,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:28,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:29,320 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:29,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:30,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:31,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:31,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:32,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:33,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:34,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:35,719 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:36,695 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:36,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:37,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:38,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:39,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:40,356 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:58:40,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:40,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:40,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:41,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:41,843 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:42,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:43,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:44,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:45,326 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:45,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:46,720 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:47,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:47,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:48,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:49,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:50,721 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:51,328 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:58:51,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:52,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:53,696 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:53,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:54,722 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:55,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:58:55,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:58:55,731 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:56,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:57,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:58,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:58:58,772 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:58:59,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:00,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:01,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:02,723 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:03,729 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:03,888 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:03,932 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:03,932 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 with policy now +2025-01-21 20:59:03,935 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:03,935 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:03,935 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:03,953 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:04,203 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/xcloc3yc-media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:59:04,667 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/6a9qdti2-media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 20:59:04,734 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:05,332 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:05,748 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:06,736 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:07,733 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:08,735 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:09,334 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:09,617 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:09,738 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:10,357 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:59:10,688 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:10,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:10,735 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:11,334 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:11,739 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:12,738 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:13,745 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:14,742 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:14,751 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:15,742 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:16,741 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:17,743 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:18,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:19,025 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:19,026 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:19,026 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:19,337 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:19,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:19,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:19,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:20,605 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:20,747 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:21,338 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:21,749 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:22,748 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:23,339 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:23,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:24,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:25,339 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:25,689 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:25,689 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:25,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:25,797 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:26,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:27,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:28,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:29,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:30,755 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:31,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:31,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:32,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:33,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:34,756 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:35,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:36,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:36,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:37,757 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:38,786 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:39,344 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:39,624 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:39,624 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 with policy now +2025-01-21 20:59:39,628 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:39,628 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:39,629 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:39,629 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:39,786 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:39,882 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/23e9vvo8-media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,344 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:40,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval +2025-01-21 20:59:40,366 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 20:59:40,647 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/v9dsyv5q-media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 20:59:40,699 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:40,700 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:40,788 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:41,345 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:41,795 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:41,808 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:42,799 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:43,793 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:44,799 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:45,346 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:45,796 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:46,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:47,061 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:47,061 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:47,061 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:47,061 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:47,086 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:59:47,086 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:59:47,238 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:47,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table +2025-01-21 20:59:47,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval +2025-01-21 20:59:47,545 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/qdrrd83r-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:47,556 INFO wandb-upload_1:2643314 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa2qxpetd +2025-01-21 20:59:47,616 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/edae6jwi-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:47,644 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 20:59:47,654 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 20:59:47,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:48,034 INFO SenderThread:2643314 [sender.py:send_request_log_artifact():1525] logged artifact run-37t5ymwr-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzNjk0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4OTg3NA==', 'latestArtifact': None}} +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:48,035 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:48,035 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,035 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,036 DEBUG SenderThread:2643314 [sender.py:send():379] send: telemetry +2025-01-21 20:59:48,036 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,039 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,040 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,040 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,040 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,041 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,041 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,042 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,042 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,042 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,043 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,043 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,043 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,044 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,044 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,044 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,044 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,045 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:48,045 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 20:59:48,299 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/codu6ha8-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 20:59:48,347 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:48,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval +2025-01-21 20:59:48,510 INFO wandb-upload_0:2643314 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdiqzwyf7 +2025-01-21 20:59:48,528 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/vcx0yrho-media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:48,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:48,926 INFO SenderThread:2643314 [sender.py:send_request_log_artifact():1525] logged artifact run-37t5ymwr-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzYzNjk2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE4OTg4Mw==', 'latestArtifact': None}} +2025-01-21 20:59:48,926 DEBUG SenderThread:2643314 [sender.py:send():379] send: files +2025-01-21 20:59:48,926 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json with policy now +2025-01-21 20:59:48,927 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 20:59:48,927 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 20:59:48,927 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:59:49,238 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/4o86scus-media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 20:59:49,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 20:59:49,348 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 20:59:49,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:50,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:51,802 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:52,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:52,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:53,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:54,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:55,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:59:55,690 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:59:55,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:56,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:57,773 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:59:57,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:58,352 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 20:59:58,803 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:59:59,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:00,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:01,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:02,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:03,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:03,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:04,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:05,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:06,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:07,804 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:08,702 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:08,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:09,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:10,358 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 21:00:10,690 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:00:10,691 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:00:10,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:11,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:12,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:13,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:13,840 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:14,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:15,805 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:16,834 DEBUG SenderThread:2643314 [sender.py:send():379] send: exit +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():588] handling runtime: 246 +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:16,834 INFO SenderThread:2643314 [sender.py:send_exit():594] send defer +2025-01-21 21:00:16,834 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,834 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:00:16,835 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:00:16,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:00:16,835 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:00:16,835 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:00:16,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:00:16,835 INFO HandlerThread:2643314 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:00:16,835 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:00:16,835 DEBUG SystemMonitor:2643314 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:00:16,836 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:00:16,836 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:00:16,889 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:00:16,890 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:00:16,890 INFO HandlerThread:2643314 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:00:16,890 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,890 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:00:16,890 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:00:16,890 DEBUG SenderThread:2643314 [sender.py:send():379] send: stats +2025-01-21 21:00:16,891 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,891 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:00:16,891 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:16,891 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:00:16,891 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:00:16,891 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:16,891 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:00:17,359 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:19,360 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:23,067 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:23,067 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:23,068 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,069 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:00:23,069 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send():379] send: history +2025-01-21 21:00:23,069 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:00:23,069 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,069 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:00:23,071 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:23,071 DEBUG SenderThread:2643314 [sender.py:send():379] send: summary +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:00:23,075 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:00:23,075 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,075 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:00:23,075 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,075 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:00:23,076 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:00:23,076 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:00:23,076 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:23,076 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:00:23,076 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:23,076 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:00:23,362 INFO Thread-12 :2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:23,835 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:26,491 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:00:26,491 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:26,492 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:26,492 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:00:26,492 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:26,492 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:00:26,492 INFO SenderThread:2643314 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:00:26,494 INFO SenderThread:2643314 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:00:26,496 INFO SenderThread:2643314 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:00:26,509 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:00:26,509 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:26,509 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:00:26,509 DEBUG SenderThread:2643314 [sender.py:send():379] send: artifact +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:00:26,770 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:00:26,770 INFO SenderThread:2643314 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:00:26,836 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml config.yaml +2025-01-21 21:00:27,363 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json wandb-summary.json +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt requirements.txt +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:00:27,364 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log output.log +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/diff.patch diff.patch +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 media/videos/eval/1_eval_1_52f38aea11256dff566a.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 media/videos/eval/2_eval_2_02b7b0943e5ffceb2ddf.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 media/videos/eval/4_eval_4_e88bcd3f25d83f27b5d9.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 media/videos/eval/0_eval_0_2f683e3d27762b787fec.mp4 +2025-01-21 21:00:27,366 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 media/videos/eval/3_eval_3_eb1f7174555f6baf68df.mp4 +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json media/table/eval/total_success_rate_ci_6_0c9cd2f25aeded733b20.table.json +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:00:27,367 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:00:27,367 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:27,367 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,367 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:00:27,373 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,373 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:00:27,373 INFO SenderThread:2643314 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:00:27,571 INFO wandb-upload_1:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/config.yaml +2025-01-21 21:00:27,607 INFO wandb-upload_0:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/wandb-summary.json +2025-01-21 21:00:27,614 INFO wandb-upload_4:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/output.log +2025-01-21 21:00:27,614 INFO wandb-upload_3:2643314 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/files/requirements.txt +2025-01-21 21:00:27,651 INFO wandb-upload_2:2643314 [upload_job.py:push():130] Uploaded file /tmp/tmphymvqtsywandb/l08znhm2-events.out.tfevents.1737510970.leonmkim-ROG-Strix-G15CS-G15CS.2643122.0 +2025-01-21 21:00:27,836 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:27,836 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:27,851 INFO Thread-11 (_thread_body):2643314 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:00:27,851 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,851 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:00:27,851 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,852 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:00:27,852 INFO SenderThread:2643314 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:00:27,853 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:27,853 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:00:27,853 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:00:27,853 INFO SenderThread:2643314 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:00:28,051 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:28,051 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:00:28,051 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:00:28,051 INFO SenderThread:2643314 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:00:28,051 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:00:28,051 INFO HandlerThread:2643314 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send():379] send: final +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send():379] send: footer +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: defer +2025-01-21 21:00:28,052 INFO SenderThread:2643314 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:00:28,052 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:28,052 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:28,053 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:00:28,053 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:00:28,055 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:00:28,056 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:00:28,100 DEBUG SenderThread:2643314 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:00:28,100 INFO MainThread:2643314 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:00:28,101 INFO MainThread:2643314 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:00:28,101 INFO MainThread:2643314 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:00:28,101 DEBUG HandlerThread:2643314 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:00:28,101 INFO HandlerThread:2643314 [handler.py:finish():882] shutting down handler +2025-01-21 21:00:29,055 INFO WriterThread:2643314 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [sender.py:finish():1615] shutting down sender +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:00:29,100 INFO SenderThread:2643314 [file_pusher.py:join():175] waiting for file pusher diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug.log b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1148fa4ad184d770901ef26822b9ae008b6ab73d --- /dev/null +++ b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Configure stats pid to 2643122 +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/settings +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:56:09,984 INFO MainThread:2643122 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug.log +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/logs/debug-internal.log +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr'} +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():619] starting backend +2025-01-21 20:56:09,985 INFO MainThread:2643122 [wandb_init.py:init():623] setting up manager +2025-01-21 20:56:09,989 INFO MainThread:2643122 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:56:09,991 INFO MainThread:2643122 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:56:10,003 INFO MainThread:2643122 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:56:10,010 INFO MainThread:2643122 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:56:10,272 INFO MainThread:2643122 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:56:10,340 INFO MainThread:2643122 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:56:10,341 INFO MainThread:2643122 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:56:10,687 INFO MainThread:2643122 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:56:10,688 INFO MainThread:2643122 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:56:10,693 INFO MainThread:2643122 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:56:10,693 INFO MainThread:2643122 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/37t5ymwr/tb, True +2025-01-21 20:56:14,612 INFO MainThread:2643122 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:00:29,102 WARNING MsgRouterThr:2643122 [router.py:message_loop():77] message_loop has been closed diff --git a/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8181777da1d96db5018d9bf0de38a81551cd3077 Binary files /dev/null and b/37t5ymwr/wandb/run-20250121_205609-37t5ymwr/run-37t5ymwr.wandb differ diff --git a/5z33ibf5/episode_rosbags/aligned_depth_to_color_K.npy b/5z33ibf5/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5z33ibf5/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5z33ibf5/episode_rosbags/cam_tf_world.npy b/5z33ibf5/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/5z33ibf5/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/5z33ibf5/episode_rosbags/color_K.npy b/5z33ibf5/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5z33ibf5/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5z33ibf5/episode_rosbags/depth_K.npy b/5z33ibf5/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/5z33ibf5/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/5z33ibf5/eval_robot.log b/5z33ibf5/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c4e955133610f88aa3ede980fbb2358a0ca61659 --- /dev/null +++ b/5z33ibf5/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:18:36,450][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:18:36,453][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/5z33ibf5/wandb/debug-internal.log b/5z33ibf5/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b1d79f9d3b94d54a45d31496dd5e6b814489bc96 --- /dev/null +++ b/5z33ibf5/wandb/debug-internal.log @@ -0,0 +1,879 @@ +2025-01-21 21:18:42,339 INFO StreamThr :2648942 [internal.py:wandb_internal():85] W&B internal server running at pid: 2648942, started at: 2025-01-21 21:18:42.339227 +2025-01-21 21:18:42,340 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:18:42,352 INFO WriterThread:2648942 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb +2025-01-21 21:18:42,352 DEBUG SenderThread:2648942 [sender.py:send():379] send: header +2025-01-21 21:18:42,359 DEBUG SenderThread:2648942 [sender.py:send():379] send: run +2025-01-21 21:18:42,657 INFO SenderThread:2648942 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files +2025-01-21 21:18:42,657 INFO SenderThread:2648942 [sender.py:_start_run_threads():1188] run started: 5z33ibf5 with start time 1737512322.339825 +2025-01-21 21:18:42,678 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:18:42,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:18:42,757 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:18:42,762 DEBUG HandlerThread:2648942 [system_info.py:__init__():26] System info init +2025-01-21 21:18:42,762 DEBUG HandlerThread:2648942 [system_info.py:__init__():41] System info init done +2025-01-21 21:18:42,762 INFO HandlerThread:2648942 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:18:42,762 INFO SystemMonitor:2648942 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:18:42,762 INFO HandlerThread:2648942 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:18:42,763 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:18:42,763 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:18:42,764 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:18:42,764 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:18:42,765 INFO SystemMonitor:2648942 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:18:42,776 DEBUG HandlerThread:2648942 [system_info.py:probe():152] Probing system +2025-01-21 21:18:42,777 DEBUG HandlerThread:2648942 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:probe():200] Probing system done +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:18:42.776324', 'startedAt': '2025-01-22T02:18:42.329441', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.98625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.942, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 357.1996383666992}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:18:42,782 INFO HandlerThread:2648942 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:18:42,782 INFO HandlerThread:2648942 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:18:42,782 DEBUG HandlerThread:2648942 [system_info.py:_save_code():44] Saving code +2025-01-21 21:18:42,784 DEBUG HandlerThread:2648942 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:18:42,784 DEBUG HandlerThread:2648942 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:18:42,846 DEBUG HandlerThread:2648942 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:18:42,846 INFO HandlerThread:2648942 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:18:42,849 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:18:42,849 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:18:42,849 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:18:42,850 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:18:43,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:18:43,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:18:43,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:43,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:43,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:43,117 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/jyqneicx-wandb-metadata.json +2025-01-21 21:18:43,123 INFO wandb-upload_2:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/oow9pu73-diff.patch +2025-01-21 21:18:43,157 INFO HandlerThread:2648942 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/tb" + save: true +} +control { + relay_id: "aea2f715713e4a8594b30a5fe7dc8655" +} +_info { + stream_id: "5z33ibf5" +} + +2025-01-21 21:18:43,157 DEBUG HandlerThread:2648942 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:18:43,169 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/zwj3yun0-code/FISH/eval_robot.py +2025-01-21 21:18:43,234 DEBUG SenderThread:2648942 [sender.py:send():379] send: telemetry +2025-01-21 21:18:43,234 DEBUG SenderThread:2648942 [sender.py:send():379] send: tbrecord +2025-01-21 21:18:43,235 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:18:43,235 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 with policy live +2025-01-21 21:18:43,433 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/ppifows0-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:18:43,658 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code +2025-01-21 21:18:43,659 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH +2025-01-21 21:18:44,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:45,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:45,658 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:18:46,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:47,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:48,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:48,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:49,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:50,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:51,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:52,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:53,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:53,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:54,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:55,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:56,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:57,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:58,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:18:58,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:18:58,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:18:58,230 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:18:59,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:00,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:01,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:02,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:03,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:04,066 DEBUG SenderThread:2648942 [sender.py:send():379] send: config +2025-01-21 21:19:04,066 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:04,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:05,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:05,663 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:06,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:07,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:07,664 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:08,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:09,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:09,664 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:09,845 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:10,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:11,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:11,665 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:12,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:13,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:13,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:13,666 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:14,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:15,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:15,200 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:15,666 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:19:16,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:17,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:18,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:19,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:20,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:21,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:21,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:22,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:23,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:23,668 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:24,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:25,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:26,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:26,636 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:27,121 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:27,669 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:28,114 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:28,114 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:28,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:29,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:30,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:31,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:32,122 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:32,260 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:33,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:33,671 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:19:34,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:35,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:36,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:37,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:38,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:38,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:39,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:40,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:41,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:42,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:42,765 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:19:42,766 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:19:43,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:43,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:43,253 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:44,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:45,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:46,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:47,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:48,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:49,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:49,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:50,125 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:51,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:52,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:53,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:54,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:19:54,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:55,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:56,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:57,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:58,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:19:58,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:19:58,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:59,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:19:59,183 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:00,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:01,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:01,313 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:20:01,314 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 with policy now +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:20:01,314 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:20:01,315 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:20:01,551 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/j2rtqp81-media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:01,679 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:20:01,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos +2025-01-21 21:20:01,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media +2025-01-21 21:20:01,982 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/by9g8eas-media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:20:02,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:03,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:03,680 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:04,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:04,903 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:05,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:06,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:07,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:07,681 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:08,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:09,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:09,682 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:10,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:10,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:11,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:12,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:12,766 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:20:13,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:13,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:14,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:14,683 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:20:14,935 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/owyry9cl-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:20:15,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:15,639 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:15,684 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:16,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:17,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:17,684 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:18,129 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:19,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:19,685 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:20,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:21,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:21,465 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:21,686 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:22,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:23,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:24,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:25,130 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:26,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:27,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:27,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:28,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:28,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:28,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:29,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:30,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:31,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:32,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:32,253 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:33,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:34,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:35,131 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:36,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:37,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:38,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:38,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:39,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:40,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:41,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:42,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:42,767 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:20:43,115 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:43,115 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:43,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:44,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:45,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:46,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:47,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:47,694 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:48,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:49,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:49,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:50,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:51,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:52,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:53,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:53,696 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:54,135 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:54,384 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:20:54,429 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:20:54,429 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 with policy now +2025-01-21 21:20:54,430 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:20:54,430 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:20:54,431 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:20:54,432 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:20:54,672 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/tgaverpi-media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:54,696 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:20:54,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:54,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:20:55,002 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/1o8zgjky-media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:20:55,135 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:55,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:56,138 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:56,697 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:20:57,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:58,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:20:58,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:20:58,140 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:59,140 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:20:59,698 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:00,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:00,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:01,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:01,699 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:02,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:03,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:04,142 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:05,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:06,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:06,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:07,142 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:07,674 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:21:07,674 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:21:07,675 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:21:07,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:21:07,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:21:08,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:09,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:09,701 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:10,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:11,143 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:11,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:11,702 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:12,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:12,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:21:13,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:13,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:13,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:13,703 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:14,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:15,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:16,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:16,254 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:17,144 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:18,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:19,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:20,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:21,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:21,706 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:22,050 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:22,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:23,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:24,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:25,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:25,707 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:26,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:27,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:27,746 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:28,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:28,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:28,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:29,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:30,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:31,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:31,709 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:32,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:33,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:33,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:34,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:35,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:36,146 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:37,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:38,119 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:38,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:39,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:40,147 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:41,148 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:42,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:42,768 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:21:43,116 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:43,116 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:43,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:43,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:44,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:45,149 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:46,150 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:47,115 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:21:47,116 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 with policy now +2025-01-21 21:21:47,118 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:21:47,119 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:21:47,119 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:21:47,138 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:21:47,150 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:47,354 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/vpnbad59-media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:47,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:21:48,007 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/vjkzjv3s-media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:21:48,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:48,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:49,123 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:49,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:49,715 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:50,164 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:51,152 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:51,716 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:52,158 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:53,158 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:54,163 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:54,597 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:21:55,159 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:55,717 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:21:56,160 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:57,163 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:58,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:21:58,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:21:58,161 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:21:59,164 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:00,170 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:00,198 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:01,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:02,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:03,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:04,167 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:05,166 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:06,128 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:06,166 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:07,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:08,169 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:09,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:10,167 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:11,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:12,127 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:12,168 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:12,768 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:22:13,145 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:13,146 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:13,174 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:13,456 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:22:13,456 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:22:13,457 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:22:13,723 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:22:13,724 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:22:13,983 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/khidhdah-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:22:14,193 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:15,181 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:15,724 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:16,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:17,181 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:17,725 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:17,941 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:18,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:19,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:19,725 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:20,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:21,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:22,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:23,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:23,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:24,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:25,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:25,727 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:26,176 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:27,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:27,728 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:28,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:28,117 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:28,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:28,250 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:29,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:30,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:31,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:32,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:33,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:34,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:34,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:35,177 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:36,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:37,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:38,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:39,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:40,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:40,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:41,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:42,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:42,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:22:43,117 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:43,117 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:43,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:44,178 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:45,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:45,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:46,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:47,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:48,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:49,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:50,179 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:51,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:51,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:52,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:53,180 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:53,737 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:54,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:54,467 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:22:54,467 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 with policy now +2025-01-21 21:22:54,470 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:22:54,470 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:22:54,470 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:22:54,486 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:22:54,706 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/bdtb2yzm-media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:54,737 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:22:54,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:54,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:22:55,025 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/x0695abo-media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:22:55,191 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:55,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:56,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:56,507 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:22:56,738 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:22:57,195 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:58,157 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:22:58,157 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:22:58,192 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:59,212 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:22:59,739 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:00,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:01,193 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:01,740 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:01,945 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:02,194 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:03,199 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:04,201 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:05,205 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:06,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:07,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:07,209 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:08,210 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:08,525 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:23:08,525 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:23:08,526 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:23:08,742 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:23:08,742 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:23:09,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:09,743 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:10,212 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:11,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:11,743 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:12,153 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:12,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:12,769 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:23:13,124 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:13,124 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:13,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:13,744 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:14,214 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:15,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:15,745 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:16,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:17,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:17,423 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:18,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:19,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:20,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:21,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:22,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:23,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:23,215 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:24,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:25,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:26,216 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:27,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:28,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:28,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:28,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:28,272 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:29,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:30,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:31,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:32,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:33,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:34,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:34,217 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:35,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:36,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:37,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:38,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:39,218 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:40,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:40,222 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:41,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:42,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:42,770 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:23:43,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:43,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:43,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:44,220 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:45,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:45,529 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:45,756 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:46,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:47,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:48,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:49,221 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:49,757 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:50,231 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:50,251 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:23:50,251 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 with policy now +2025-01-21 21:23:50,297 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:23:50,298 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:23:50,298 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:23:50,311 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:23:50,481 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/e6vqeati-media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:50,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval +2025-01-21 21:23:51,072 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/ajfhb57u-media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:23:51,236 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:51,312 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:51,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:52,226 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:52,758 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:53,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:54,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:55,246 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:55,759 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:56,245 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:56,695 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:23:57,232 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:57,760 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:23:58,126 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:23:58,127 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:23:58,238 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:23:59,243 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:00,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:00,588 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:00,588 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:00,589 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:00,761 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:00,761 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:00,804 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:24:00,804 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:24:01,037 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/rmqtnflz-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:01,065 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:24:01,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:01,260 INFO wandb-upload_0:2648942 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa_4wwwtd +2025-01-21 21:24:01,449 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:24:01,460 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:24:01,676 INFO SenderThread:2648942 [sender.py:send_request_log_artifact():1525] logged artifact run-5z33ibf5-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY2NTU2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NDk1MQ==', 'latestArtifact': None}} +2025-01-21 21:24:01,676 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:24:01,676 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-21 21:24:01,677 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:01,677 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,677 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send():379] send: telemetry +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,678 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,678 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,679 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,679 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,681 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,681 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,682 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,682 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,682 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,682 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,683 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,683 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,683 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,684 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,684 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,684 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,684 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,685 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:01,685 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:01,685 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media +2025-01-21 21:24:01,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval +2025-01-21 21:24:01,763 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table +2025-01-21 21:24:01,881 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/84vvl9wq-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:01,965 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/2rogxjlf-media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:02,103 INFO wandb-upload_2:2648942 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2svt4rn5 +2025-01-21 21:24:02,138 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/h2m904lt-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:02,239 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:02,467 INFO SenderThread:2648942 [sender.py:send_request_log_artifact():1525] logged artifact run-5z33ibf5-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY2NTU4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NDk1OQ==', 'latestArtifact': None}} +2025-01-21 21:24:02,467 DEBUG SenderThread:2648942 [sender.py:send():379] send: files +2025-01-21 21:24:02,467 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json with policy now +2025-01-21 21:24:02,468 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:02,468 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:02,468 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:02,469 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:02,712 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/rau7ntk6-media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:02,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:03,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:03,762 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:04,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:05,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:06,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:07,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:08,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:08,240 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:09,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:10,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:11,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:12,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:12,770 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:24:13,120 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:24:13,120 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:24:13,192 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:13,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:13,766 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:24:14,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:15,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:16,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:17,241 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:18,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:19,141 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:19,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:20,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:21,242 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:21,981 DEBUG SenderThread:2648942 [sender.py:send():379] send: exit +2025-01-21 21:24:21,981 INFO SenderThread:2648942 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:24:21,981 INFO SenderThread:2648942 [sender.py:send_exit():588] handling runtime: 339 +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:send_exit():594] send defer +2025-01-21 21:24:21,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,982 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:24:21,982 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:24:21,982 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:24:21,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,982 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:24:21,983 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:21,983 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:24:21,983 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:24:21,983 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:24:21,983 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:24:21,983 DEBUG SystemMonitor:2648942 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:24:21,983 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:24:21,984 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:24:22,059 INFO HandlerThread:2648942 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send():379] send: stats +2025-01-21 21:24:22,060 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:22,060 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:24:22,060 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:24:22,060 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:24:22,061 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:22,061 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:24:22,770 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:23,770 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:28,596 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:28,596 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:28,597 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,598 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:24:28,598 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send():379] send: history +2025-01-21 21:24:28,598 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:24:28,598 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,599 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:24:28,600 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:28,600 DEBUG SenderThread:2648942 [sender.py:send():379] send: summary +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:24:28,604 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:24:28,604 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:24:28,604 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,605 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:24:28,605 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:24:28,605 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:24:28,605 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:28,605 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:24:28,605 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:28,605 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:24:28,773 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:28,982 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:30,583 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:24:30,583 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:30,584 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:30,584 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:24:30,584 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:30,584 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:24:30,584 INFO SenderThread:2648942 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:24:30,587 INFO SenderThread:2648942 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:24:30,589 INFO SenderThread:2648942 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:24:30,598 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:24:30,598 DEBUG SenderThread:2648942 [sender.py:send():379] send: artifact +2025-01-21 21:24:30,601 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:30,601 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:24:30,773 INFO Thread-12 :2648942 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:24:30,812 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:24:30,812 INFO SenderThread:2648942 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:24:30,983 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml config.yaml +2025-01-21 21:24:31,774 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json wandb-summary.json +2025-01-21 21:24:31,775 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt requirements.txt +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:24:31,776 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log output.log +2025-01-21 21:24:31,777 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/diff.patch diff.patch +2025-01-21 21:24:31,782 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:24:31,782 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 media/videos/eval/2_eval_2_1bfe16d417bf0eae1e35.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 media/videos/eval/0_eval_0_52952b9d92244b702210.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 media/videos/eval/4_eval_4_0b307d108c1b40ec3a03.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 media/videos/eval/1_eval_1_f36a449d922a9d3cb168.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 media/videos/eval/3_eval_3_d2f247956aef8b976274.mp4 +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json media/table/eval/total_success_rate_ci_6_7f69fec0f5489086ad7f.table.json +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-21 21:24:31,783 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:24:31,783 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:31,783 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:31,783 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:24:31,784 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:31,784 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:24:31,784 INFO SenderThread:2648942 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:24:31,968 INFO wandb-upload_1:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/config.yaml +2025-01-21 21:24:31,969 INFO wandb-upload_0:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/wandb-summary.json +2025-01-21 21:24:31,984 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:31,984 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:32,009 INFO wandb-upload_2:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt +2025-01-21 21:24:32,018 INFO wandb-upload_4:2648942 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/output.log +2025-01-21 21:24:32,645 INFO wandb-upload_3:2648942 [upload_job.py:push():130] Uploaded file /tmp/tmpnok3f10zwandb/f6ont0lf-events.out.tfevents.1737512323.leonmkim-ROG-Strix-G15CS-G15CS.2648747.0 +2025-01-21 21:24:32,845 INFO Thread-11 (_thread_body):2648942 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:24:32,845 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:32,845 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:24:32,846 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:32,846 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:24:32,846 INFO SenderThread:2648942 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:24:32,847 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:24:32,847 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:32,847 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:24:32,847 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:32,848 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:24:32,848 INFO SenderThread:2648942 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:24:32,984 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,132 INFO SenderThread:2648942 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:24:33,132 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:24:33,132 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,132 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:33,132 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:24:33,133 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:24:33,133 INFO HandlerThread:2648942 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send():379] send: final +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send():379] send: footer +2025-01-21 21:24:33,133 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: defer +2025-01-21 21:24:33,133 INFO SenderThread:2648942 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:24:33,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,134 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,134 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:24:33,135 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:24:33,136 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:24:33,136 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:24:33,137 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:24:33,137 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:24:33,206 DEBUG SenderThread:2648942 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:24:33,206 INFO MainThread:2648942 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:24:33,207 INFO MainThread:2648942 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:24:33,207 INFO MainThread:2648942 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:24:33,207 DEBUG HandlerThread:2648942 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:24:33,207 INFO HandlerThread:2648942 [handler.py:finish():882] shutting down handler +2025-01-21 21:24:34,136 INFO WriterThread:2648942 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [sender.py:finish():1615] shutting down sender +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:24:34,206 INFO SenderThread:2648942 [file_pusher.py:join():175] waiting for file pusher diff --git a/5z33ibf5/wandb/debug.log b/5z33ibf5/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6b7096dafa2e4d7df4475e94f7d018845360e01a --- /dev/null +++ b/5z33ibf5/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Configure stats pid to 2648747 +2025-01-21 21:18:42,331 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/settings +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug.log +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/logs/debug-internal.log +2025-01-21 21:18:42,332 INFO MainThread:2648747 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5'} +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():619] starting backend +2025-01-21 21:18:42,333 INFO MainThread:2648747 [wandb_init.py:init():623] setting up manager +2025-01-21 21:18:42,338 INFO MainThread:2648747 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:18:42,339 INFO MainThread:2648747 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:18:42,351 INFO MainThread:2648747 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:18:42,357 INFO MainThread:2648747 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:18:42,678 INFO MainThread:2648747 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:18:42,752 INFO MainThread:2648747 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:18:42,752 INFO MainThread:2648747 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:18:43,114 INFO MainThread:2648747 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:18:43,115 INFO MainThread:2648747 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:18:43,118 INFO MainThread:2648747 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:18:43,118 INFO MainThread:2648747 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/5z33ibf5/tb, True +2025-01-21 21:19:04,065 INFO MainThread:2648747 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:24:34,208 WARNING MsgRouterThr:2648747 [router.py:message_loop():77] message_loop has been closed diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..00df6324acf806091e62d1aa7176c4cdd1f1a533 Binary files /dev/null and b/5z33ibf5/wandb/run-20250121_211842-5z33ibf5/run-5z33ibf5.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e42c81c485510fd4d9e15c5d399f10b0ad72649b --- /dev/null +++ b/config.yaml @@ -0,0 +1,575 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '4295_0' +wandb_notes: 4295_0_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/hcp6xxlt/.hydra/config.yaml b/hcp6xxlt/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc75d61dbe4b7ba0ede1ab77029649cc5b18502 --- /dev/null +++ b/hcp6xxlt/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hcp6xxlt/.hydra/hydra.yaml b/hcp6xxlt/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce9d02c46a3ef9ca5510ad65f3be17d6e47cc7ec --- /dev/null +++ b/hcp6xxlt/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hcp6xxlt/.hydra/overrides.yaml b/hcp6xxlt/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hcp6xxlt/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hcp6xxlt/episode_rosbags/aligned_depth_to_color_K.npy b/hcp6xxlt/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hcp6xxlt/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hcp6xxlt/episode_rosbags/cam_tf_world.npy b/hcp6xxlt/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hcp6xxlt/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hcp6xxlt/episode_rosbags/color_K.npy b/hcp6xxlt/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hcp6xxlt/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hcp6xxlt/episode_rosbags/depth_K.npy b/hcp6xxlt/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hcp6xxlt/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hcp6xxlt/eval_robot.log b/hcp6xxlt/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..062d32d0d6f226c212d32f081bbff5c154acd498 --- /dev/null +++ b/hcp6xxlt/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:35:46,625][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:35:46,628][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hcp6xxlt/eval_video/0_eval.mp4 b/hcp6xxlt/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fdc67a74d6a150d6579ff4dc69fbbc0c1296bf6b --- /dev/null +++ b/hcp6xxlt/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5a262079efd287fc7752209af804f395b8b6fa55df2127684daf2ea2e3affb2 +size 714627 diff --git a/hcp6xxlt/eval_video/1_eval.mp4 b/hcp6xxlt/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..713d1c0f6781c114c27f880ab91044eaee746612 --- /dev/null +++ b/hcp6xxlt/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81d19fe2171d168a019fed6c427cfc3e60c80f0cce2358b2aee9be74363cd13 +size 806848 diff --git a/hcp6xxlt/eval_video/2_eval.mp4 b/hcp6xxlt/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7ee30fd84bda2cee797b53c746a0ce94898c9bf --- /dev/null +++ b/hcp6xxlt/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4db46f72773c3d5ecad693d663dfc905569e7ba33e61290453adb4070851aa +size 721038 diff --git a/hcp6xxlt/eval_video/3_eval.mp4 b/hcp6xxlt/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3611a5c26976c6c982308d54385cea8a5661203 --- /dev/null +++ b/hcp6xxlt/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b67859ddc474a3c0abd2ee74f372260b4d45da37f4f54fdd6a585d1d38b3f3 +size 837111 diff --git a/hcp6xxlt/eval_video/4_eval.mp4 b/hcp6xxlt/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a002648262b49115aa0ff048410752b9cd941c1f --- /dev/null +++ b/hcp6xxlt/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d14611aa0de1dd53b11075930c33006a1f467d250697a0acdfbf81152c676611 +size 1583825 diff --git a/hcp6xxlt/tb/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 b/hcp6xxlt/tb/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 new file mode 100644 index 0000000000000000000000000000000000000000..f2f12d199275914092977769dae4bf6aaf939633 --- /dev/null +++ b/hcp6xxlt/tb/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:302870f4b2b75728df7a24f121a898c1896e759589d338d514a62ca92c042b88 +size 6499 diff --git a/hcp6xxlt/wandb/debug-internal.log b/hcp6xxlt/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46e5992b38e95739880019d6d6c763bfaca633c2 --- /dev/null +++ b/hcp6xxlt/wandb/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-21 21:35:52,330 INFO StreamThr :2653360 [internal.py:wandb_internal():85] W&B internal server running at pid: 2653360, started at: 2025-01-21 21:35:52.330081 +2025-01-21 21:35:52,331 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:35:52,346 INFO WriterThread:2653360 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb +2025-01-21 21:35:52,346 DEBUG SenderThread:2653360 [sender.py:send():379] send: header +2025-01-21 21:35:52,354 DEBUG SenderThread:2653360 [sender.py:send():379] send: run +2025-01-21 21:35:52,577 INFO SenderThread:2653360 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files +2025-01-21 21:35:52,577 INFO SenderThread:2653360 [sender.py:_start_run_threads():1188] run started: hcp6xxlt with start time 1737513352.330422 +2025-01-21 21:35:52,592 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:35:52,593 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:35:52,668 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:35:52,673 DEBUG HandlerThread:2653360 [system_info.py:__init__():26] System info init +2025-01-21 21:35:52,673 DEBUG HandlerThread:2653360 [system_info.py:__init__():41] System info init done +2025-01-21 21:35:52,673 INFO HandlerThread:2653360 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:35:52,673 INFO SystemMonitor:2653360 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:35:52,673 INFO HandlerThread:2653360 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:35:52,674 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:35:52,675 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:35:52,687 DEBUG HandlerThread:2653360 [system_info.py:probe():152] Probing system +2025-01-21 21:35:52,689 DEBUG HandlerThread:2653360 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_info.py:probe():200] Probing system done +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:35:52.687764', 'startedAt': '2025-01-22T02:35:52.322613', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.96425, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.806, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 392.2366714477539}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:35:52,694 INFO HandlerThread:2653360 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:35:52,694 INFO HandlerThread:2653360 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:35:52,694 DEBUG HandlerThread:2653360 [system_info.py:_save_code():44] Saving code +2025-01-21 21:35:52,695 DEBUG HandlerThread:2653360 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:35:52,695 DEBUG HandlerThread:2653360 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:35:52,759 DEBUG HandlerThread:2653360 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:35:52,760 INFO HandlerThread:2653360 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:35:52,763 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:35:52,763 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:35:52,763 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:35:52,764 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:35:53,001 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:35:53,002 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:35:53,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:53,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:53,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:53,005 INFO HandlerThread:2653360 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/tb" + save: true +} +control { + relay_id: "55dd0b2c1b1743ffb18214d1539a47a8" +} +_info { + stream_id: "hcp6xxlt" +} + +2025-01-21 21:35:53,005 DEBUG HandlerThread:2653360 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:35:53,022 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/njokud3o-code/FISH/eval_robot.py +2025-01-21 21:35:53,062 DEBUG SenderThread:2653360 [sender.py:send():379] send: telemetry +2025-01-21 21:35:53,062 DEBUG SenderThread:2653360 [sender.py:send():379] send: tbrecord +2025-01-21 21:35:53,063 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:35:53,063 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 with policy live +2025-01-21 21:35:53,068 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/raznx8jb-diff.patch +2025-01-21 21:35:53,133 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/qrzo0zai-wandb-metadata.json +2025-01-21 21:35:53,259 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/6hrrc4bp-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code +2025-01-21 21:35:54,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:55,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:55,578 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:56,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:56,324 DEBUG SenderThread:2653360 [sender.py:send():379] send: config +2025-01-21 21:35:57,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:57,335 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:57,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:58,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:59,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:59,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:00,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:01,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:01,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:02,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:03,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:03,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:03,581 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:04,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:05,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:05,582 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:06,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:07,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:07,582 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:08,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:08,002 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:08,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:08,091 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:09,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:10,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:11,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:11,583 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:12,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:13,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:14,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:14,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:15,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:16,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:17,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:18,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:19,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:19,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:20,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:21,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:22,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:23,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:23,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:23,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:24,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:24,280 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:24,507 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:36:24,507 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:36:24,508 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 with policy now +2025-01-21 21:36:24,508 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:36:24,508 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:36:24,509 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media +2025-01-21 21:36:24,774 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/oji3jzi9-media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:25,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:25,041 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/glpo002s-media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:25,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:26,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:27,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:28,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:29,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:29,588 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:30,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:30,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:31,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:31,589 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:36:32,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:33,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:33,590 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:34,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:35,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:35,196 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:36,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:37,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:37,592 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:38,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:38,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:38,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:39,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:40,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:41,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:41,088 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:42,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:43,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:44,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:45,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:45,595 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:46,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:46,225 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:47,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:48,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:49,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:50,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:51,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:51,251 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:51,597 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:52,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:52,548 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:36:52,549 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 with policy now +2025-01-21 21:36:52,551 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:36:52,552 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:36:52,552 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:36:52,552 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:36:52,677 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:36:52,677 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:36:52,792 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w01589gh-media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:53,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:53,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:53,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:53,056 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/tzmlapo8-media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:53,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:54,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:54,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:55,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:56,016 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:57,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:57,237 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:57,599 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:58,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:59,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:00,016 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:01,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:01,601 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:02,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:02,717 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:03,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:03,601 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:04,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:05,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:06,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:07,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:08,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:08,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:08,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:08,117 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:09,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:10,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:11,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:12,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:13,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:14,007 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:14,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:15,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:16,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:17,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:18,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:19,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:20,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:20,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:21,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:22,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:22,678 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:37:23,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:23,005 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:23,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:24,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:25,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:25,112 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:26,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:27,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:28,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:29,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:30,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:31,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:31,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:32,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:33,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:34,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:35,026 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:36,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:37,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:37,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:38,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:38,008 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:38,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:39,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:40,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:40,614 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:37:40,873 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/gdo8dt21-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:37:41,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:41,406 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:37:41,406 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:37:41,407 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:37:41,614 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:37:41,615 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:42,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:42,581 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:43,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:43,616 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:44,026 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:45,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:45,616 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:46,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:47,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:47,617 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:47,991 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:48,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:49,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:50,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:51,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:52,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:52,678 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:37:53,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:53,004 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:53,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:53,133 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:54,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:55,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:56,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:57,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:58,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:58,684 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:59,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:59,621 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:59,867 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:37:59,867 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 with policy now +2025-01-21 21:37:59,869 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:37:59,870 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:37:59,870 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:37:59,870 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:00,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:00,110 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/xjfdismv-media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:38:00,916 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/clyopkyv-media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:01,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:01,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:02,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:03,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:04,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:04,481 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:05,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:05,624 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:06,064 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:07,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:07,624 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:08,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:08,010 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:08,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:09,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:09,223 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:09,223 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:09,236 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:09,723 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:10,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:11,063 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:11,626 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:12,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:13,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:13,626 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:14,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:15,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:15,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:15,627 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:16,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:17,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:18,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:19,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:20,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:20,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:21,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:22,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:22,679 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:38:23,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:23,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:23,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:24,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:25,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:25,133 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:26,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:27,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:28,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:29,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:30,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:30,290 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:31,067 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:31,560 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:38:31,560 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 with policy now +2025-01-21 21:38:31,562 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:38:31,563 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:31,563 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:31,563 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:38:31,816 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/1hm85y0w-media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:32,065 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:32,089 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/bbyywmp8-media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:33,067 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:33,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:34,076 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:35,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:35,833 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:36,068 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:37,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:37,635 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:38,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:38,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:38,066 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:38,635 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:39,077 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:39,260 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:39,261 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:39,261 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:40,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:41,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:41,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:41,725 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:42,078 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:43,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:43,637 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:44,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:45,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:45,638 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:46,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:47,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:47,638 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:47,721 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:48,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:49,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:50,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:51,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:52,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:52,679 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:38:53,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:53,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:53,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:53,126 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:54,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:55,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:56,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:57,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:58,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:59,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:59,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:00,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:01,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:02,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:03,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:04,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:04,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:05,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:06,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:07,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:08,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:08,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:08,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:09,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:09,124 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:10,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:11,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:12,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:13,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:14,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:15,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:15,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:16,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:17,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:18,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:19,074 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:20,098 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:20,379 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:20,458 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:20,459 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 with policy now +2025-01-21 21:39:20,465 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:20,465 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:20,465 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:20,470 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:39:20,728 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ch2imcub-media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:21,010 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w5du036o-media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:21,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:21,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:22,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:22,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:22,680 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:39:23,027 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:23,027 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:23,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:24,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:25,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:25,651 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:26,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:26,145 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:27,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:28,100 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:29,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:29,334 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:29,334 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:29,334 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:29,659 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:39:29,659 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:39:29,893 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:29,909 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/a22src9i-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:30,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:30,255 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:39:30,267 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:30,275 INFO wandb-upload_2:2653360 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9cbjc38d +2025-01-21 21:39:30,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:30,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval +2025-01-21 21:39:30,734 INFO SenderThread:2653360 [sender.py:send_request_log_artifact():1525] logged artifact run-hcp6xxlt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4NDA2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5ODM3MA==', 'latestArtifact': None}} +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:30,734 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,735 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send():379] send: telemetry +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,736 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,736 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,737 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,737 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,737 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,738 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,738 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,739 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,739 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,740 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,740 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,740 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,740 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,741 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,741 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:39:30,917 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ijswz3il-media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:30,927 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/d5y6if5t-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:31,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:31,111 INFO wandb-upload_2:2653360 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5vnbppk7 +2025-01-21 21:39:31,216 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/37wzxwli-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:31,515 INFO SenderThread:2653360 [sender.py:send_request_log_artifact():1525] logged artifact run-hcp6xxlt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4NDA4Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5ODM3Mw==', 'latestArtifact': None}} +2025-01-21 21:39:31,515 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:31,515 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json with policy now +2025-01-21 21:39:31,516 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:31,516 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:31,516 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:31,517 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:31,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:31,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:31,788 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w18vmkmu-media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:32,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:33,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:34,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:35,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:36,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:37,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:37,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:38,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:38,012 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:38,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:39,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:40,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:41,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:42,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:42,122 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:42,658 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:39:43,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:44,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:45,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:46,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:47,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:48,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:48,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:49,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:50,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:51,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:52,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:52,680 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:39:53,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:53,012 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:53,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:53,153 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:54,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:55,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:56,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:57,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:58,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:59,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:59,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:00,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:01,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:02,097 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:03,097 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:03,521 DEBUG SenderThread:2653360 [sender.py:send():379] send: exit +2025-01-21 21:40:03,521 INFO SenderThread:2653360 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:40:03,522 INFO SenderThread:2653360 [sender.py:send_exit():588] handling runtime: 250 +2025-01-21 21:40:03,522 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:send_exit():594] send defer +2025-01-21 21:40:03,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,523 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:40:03,523 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:40:03,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,523 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:40:03,524 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,524 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:40:03,524 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:40:03,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:40:03,524 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:40:03,525 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:40:03,579 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,579 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:40:03,579 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:40:03,580 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:40:03,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,580 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:40:03,580 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,580 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:40:03,580 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:40:03,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,580 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:40:03,668 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:05,670 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:40:10,344 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,345 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:40:10,345 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:40:10,345 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,346 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:40:10,346 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:10,346 DEBUG SenderThread:2653360 [sender.py:send():379] send: summary +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:10,347 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:40:10,347 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,347 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:40:10,347 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:40:10,347 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,348 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,348 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:40:10,348 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,348 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:40:10,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:10,672 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:12,601 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:40:12,601 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:12,601 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:12,602 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:40:12,602 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:12,602 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:40:12,602 INFO SenderThread:2653360 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:40:12,604 INFO SenderThread:2653360 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:40:12,606 INFO SenderThread:2653360 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:40:12,621 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:40:12,621 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:12,621 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:40:12,621 DEBUG SenderThread:2653360 [sender.py:send():379] send: artifact +2025-01-21 21:40:12,673 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:40:12,817 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:40:13,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:13,673 INFO SenderThread:2653360 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files +2025-01-21 21:40:13,673 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml config.yaml +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json wandb-summary.json +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt requirements.txt +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log output.log +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch diff.patch +2025-01-21 21:40:13,679 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:40:13,683 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:13,683 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:13,683 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:40:13,683 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:40:13,862 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:13,865 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt +2025-01-21 21:40:13,950 INFO wandb-upload_4:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:13,955 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:40:13,996 INFO wandb-upload_3:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ee06zo97-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:40:14,196 INFO Thread-11 (_thread_body):2653360 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:40:14,197 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,197 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:40:14,197 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,197 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:40:14,197 INFO SenderThread:2653360 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:40:14,198 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:40:14,198 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,198 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:40:14,199 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,199 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:40:14,199 INFO SenderThread:2653360 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:40:14,524 INFO SenderThread:2653360 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:40:14,524 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:40:14,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,525 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:40:14,525 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,525 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,525 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:40:14,525 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:40:14,526 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,526 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send():379] send: final +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send():379] send: footer +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,526 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:40:14,527 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,527 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:40:14,528 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:40:14,530 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:14,579 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,579 INFO MainThread:2653360 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:40:14,580 INFO MainThread:2653360 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:40:14,580 INFO MainThread:2653360 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:40:14,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:40:14,580 INFO HandlerThread:2653360 [handler.py:finish():882] shutting down handler +2025-01-21 21:40:15,528 INFO WriterThread:2653360 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [sender.py:finish():1615] shutting down sender +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [file_pusher.py:join():175] waiting for file pusher diff --git a/hcp6xxlt/wandb/debug.log b/hcp6xxlt/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d852ab8f745eaad1ac7907eebbc80ae5819c6cdf --- /dev/null +++ b/hcp6xxlt/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Configure stats pid to 2653170 +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/settings +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug.log +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug-internal.log +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt'} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():619] starting backend +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():623] setting up manager +2025-01-21 21:35:52,329 INFO MainThread:2653170 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:35:52,330 INFO MainThread:2653170 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:35:52,345 INFO MainThread:2653170 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:35:52,351 INFO MainThread:2653170 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:35:52,592 INFO MainThread:2653170 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:35:52,661 INFO MainThread:2653170 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:35:52,661 INFO MainThread:2653170 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:35:53,004 INFO MainThread:2653170 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:35:53,005 INFO MainThread:2653170 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/tb, True +2025-01-21 21:35:56,323 INFO MainThread:2653170 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:40:15,581 WARNING MsgRouterThr:2653170 [router.py:message_loop():77] message_loop has been closed diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1bf6fe716f18cb62bf4eae3f7735781b831dac6 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737513352 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3a323adf2e00f53813f77e99fa964046e0cc21f4 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch @@ -0,0 +1,13 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c2e87d3 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json new file mode 100644 index 0000000000000000000000000000000000000000..765ae70cb98b3f1db2a49339aad63e786981ce80 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[8, 25, 0.32, 0.1340599954860146, 0.5579540729406796, 0.1672809795210612, 0.5333203971319316, 0.18870285904672984, 0.4791559204500879, 0.2017544434071032, 0.46316119284174495]]} \ No newline at end of file diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fdc67a74d6a150d6579ff4dc69fbbc0c1296bf6b --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5a262079efd287fc7752209af804f395b8b6fa55df2127684daf2ea2e3affb2 +size 714627 diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..713d1c0f6781c114c27f880ab91044eaee746612 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a81d19fe2171d168a019fed6c427cfc3e60c80f0cce2358b2aee9be74363cd13 +size 806848 diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7ee30fd84bda2cee797b53c746a0ce94898c9bf --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4db46f72773c3d5ecad693d663dfc905569e7ba33e61290453adb4070851aa +size 721038 diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3611a5c26976c6c982308d54385cea8a5661203 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b67859ddc474a3c0abd2ee74f372260b4d45da37f4f54fdd6a585d1d38b3f3 +size 837111 diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a002648262b49115aa0ff048410752b9cd941c1f --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d14611aa0de1dd53b11075930c33006a1f467d250697a0acdfbf81152c676611 +size 1583825 diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c800bf12e857ed8c38b747009744878fc1f8ade0 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737513356.324622]: resetting environment +[INFO] [1737513356.328663]: cleared current plan +[INFO] [1737513356.329601]: moving to home +[INFO] [1737513357.334473]: reached home +[INFO] [1737513357.335900]: reset action history +[INFO] [1737513360.044595]: environment reset +Starting episode 0 +[INFO] [1737513360.049118]: resetting environment +[INFO] [1737513360.053151]: cleared current plan +[INFO] [1737513360.054010]: moving to home +[INFO] [1737513361.056351]: reached home +[INFO] [1737513361.057581]: reset action history +[INFO] [1737513363.768748]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737513365.459850]: resetting environment +[INFO] [1737513365.463778]: cleared current plan +[INFO] [1737513365.464999]: moving to home +[INFO] [1737513366.467064]: reached home +[INFO] [1737513366.468587]: reset action history +[INFO] [1737513369.177262]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/episode_rosbags/episode_0_2025-01-21-21-36-09.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2653170.log +started roslaunch server http://158.130.50.37:38899/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2653469] +process[print_text-2]: started with pid [2653470] +process[rosbag_record-3]: started with pid [2653494] +started bagging! +preempting episode +[INFO] [1737513383.508492]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737513384.275484]: Stored episode 1. +[INFO] [1737513384.276449]: Saving video... +[INFO] [1737513384.496651]: Video saved! +Starting episode 1 +[INFO] [1737513384.508938]: resetting environment +[INFO] [1737513384.520279]: cleared current plan +[INFO] [1737513384.520523]: moving to home +[INFO] [1737513388.025102]: reached home +[INFO] [1737513388.025740]: reset action history +[INFO] [1737513390.735800]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513391.479213]: resetting environment +[INFO] [1737513391.483002]: cleared current plan +[INFO] [1737513391.483196]: moving to home +[INFO] [1737513392.484635]: reached home +[INFO] [1737513392.485401]: reset action history +[INFO] [1737513395.195967]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/episode_rosbags/episode_1_2025-01-21-21-36-35.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2653170.log +started roslaunch server http://158.130.50.37:46733/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2653613] +process[print_text-5]: started with pid [2653614] +process[rosbag_record-6]: started with pid [2653615] +started bagging! +For topic gripper_width: timestamp difference is 103196211 for nearest: 1737513404179150410 - target: 1737513404075954199 at idx 108 +For topic gripper_width: timestamp difference is 107312086 for nearest: 1737513409079164865 - target: 1737513408971852779 at idx 166 +preempting episode +[INFO] [1737513411.263067]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737513411.998789]: Stored episode 2. +[INFO] [1737513412.001378]: Saving video... +[INFO] [1737513412.496155]: Video saved! +Starting episode 2 +[INFO] [1737513412.555965]: resetting environment +[INFO] [1737513412.579906]: cleared current plan +[INFO] [1737513412.580426]: moving to home +[WARN] [1737513412.407387]: Plan exhausted +[WARN] [1737513412.450418]: Plan exhausted +[WARN] [1737513412.487408]: Plan exhausted +[WARN] [1737513412.524858]: Plan exhausted +[WARN] [1737513412.564377]: Plan exhausted +[INFO] [1737513416.189159]: reached home +[INFO] [1737513416.191869]: reset action history +[INFO] [1737513418.971056]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +Invalid key press, try again +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513460.518270]: resetting environment +[INFO] [1737513460.536934]: cleared current plan +[INFO] [1737513460.538080]: moving to home +[INFO] [1737513461.538951]: reached home +[INFO] [1737513461.539620]: reset action history +[INFO] [1737513464.247713]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/episode_rosbags/episode_2_2025-01-21-21-37-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2653170.log +started roslaunch server http://158.130.50.37:40405/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2653783] +process[print_text-8]: started with pid [2653784] +process[rosbag_record-9]: started with pid [2653785] +started bagging! +For topic gripper_width: timestamp difference is 101895788 for nearest: 1737513478679152229 - target: 1737513478577256441 at idx 206 +preempting episode +[INFO] [1737513478.724428]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737513479.463821]: Stored episode 3. +[INFO] [1737513479.465575]: Saving video... +[INFO] [1737513479.818244]: Video saved! +Starting episode 3 +[INFO] [1737513479.872730]: resetting environment +[INFO] [1737513479.911377]: cleared current plan +[INFO] [1737513479.926088]: moving to home +[INFO] [1737513483.436333]: reached home +[INFO] [1737513483.440042]: reset action history +[INFO] [1737513486.237502]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513488.674527]: resetting environment +[INFO] [1737513488.706262]: cleared current plan +[INFO] [1737513488.707718]: moving to home +[INFO] [1737513489.722565]: reached home +[INFO] [1737513489.726036]: reset action history +[INFO] [1737513492.552940]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/episode_rosbags/episode_3_2025-01-21-21-38-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2653170.log +started roslaunch server http://158.130.50.37:45611/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2653899] +process[print_text-11]: started with pid [2653900] +process[rosbag_record-12]: started with pid [2653924] +started bagging! +preempting episode +[INFO] [1737513510.296686]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737513511.069276]: Stored episode 4. +[INFO] [1737513511.075085]: Saving video... +[INFO] [1737513511.516552]: Video saved! +Starting episode 4 +[INFO] [1737513511.568511]: resetting environment +[INFO] [1737513511.581643]: cleared current plan +[INFO] [1737513511.582269]: moving to home +[INFO] [1737513514.791849]: reached home +[INFO] [1737513514.800059]: reset action history +[INFO] [1737513517.597335]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513518.634607]: resetting environment +[INFO] [1737513518.670106]: cleared current plan +[INFO] [1737513518.673954]: moving to home +[INFO] [1737513519.683549]: reached home +[INFO] [1737513519.689610]: reset action history +[INFO] [1737513522.469519]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/episode_rosbags/episode_4_2025-01-21-21-38-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2653170.log +started roslaunch server http://158.130.50.37:41077/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2654061] +process[print_text-14]: started with pid [2654062] +process[rosbag_record-15]: started with pid [2654063] +started bagging! +For topic gripper_width: timestamp difference is 106678294 for nearest: 1737513525679166602 - target: 1737513525572488308 at idx 48 +[INFO] [1737513559.097194]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1737513559.871365]: Stored episode 5. +[INFO] [1737513559.878852]: Saving video... +[INFO] [1737513560.375446]: Video saved! +[INFO] [1737513560.467463]: resetting environment +[INFO] [1737513560.497520]: cleared current plan +[INFO] [1737513560.498240]: moving to home +[WARN] [1737513560.444491]: Plan exhausted +[WARN] [1737513560.484417]: Plan exhausted +[INFO] [1737513564.101644]: reached home +[INFO] [1737513564.107213]: reset action history +[INFO] [1737513566.964590]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d058df909c6237102119a4a1ccbc256cc19ab973 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:35:52.687764", + "startedAt": "2025-01-22T02:35:52.322613", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3935.96425, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.806, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 392.2366714477539 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f2583c3a81ec670a49efe7c1b7dc485703efcef3 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "b5a262079efd287fc7752209af804f395b8b6fa55df2127684daf2ea2e3affb2", "size": 714627, "path": "media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4"}, "global_step": 368, "_timestamp": 1737513570.1627667, "_runtime": 217.83234477043152, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a81d19fe2171d168a019fed6c427cfc3e60c80f0cce2358b2aee9be74363cd13", "size": 806848, "path": "media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4"}, "eval/num_success": 1.0, "episode": 25.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "7f4db46f72773c3d5ecad693d663dfc905569e7ba33e61290453adb4070851aa", "size": 721038, "path": "media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "99b67859ddc474a3c0abd2ee74f372260b4d45da37f4f54fdd6a585d1d38b3f3", "size": 837111, "path": "media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d14611aa0de1dd53b11075930c33006a1f467d250697a0acdfbf81152c676611", "size": 1583825, "path": "media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://gxj16y2n03ja2qsv0cnvqm7ul62xc30dxchn03j7lxn6pddylsch32l4vxardob1z0321yr0vt2hw58jyeyw1kxlb1dlnt7tv96v0uh00nsgw14mcpe3mkl7vjjsm0mt/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ajhojvum4p3hcw1cm7hws1woxzarpomy8kuu63yx7pjlmdrxe4caadfq268t0nh53zn70apur64j84s4kvgrqtctqht8kxt3jf16p2fhxdk7qz5e762z8bbisa1qumgt:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 8, "total_num_episodes": 25, "total_success_rate": 0.32, "total_success_rate_lb_0.01": 0.1340599954860146, "total_success_rate_ub_0.01": 0.5579540729406796, "total_success_rate_lb_0.025": 0.1672809795210612, "total_success_rate_ub_0.025": 0.5333203971319316, "total_success_rate_lb_0.05": 0.18870285904672984, "total_success_rate_ub_0.05": 0.4791559204500879, "total_success_rate_lb_0.1": 0.2017544434071032, "total_success_rate_ub_0.1": 0.46316119284174495, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "de2f891a47c3da5129844f36db644bad40dfbc03f2bfe76984ba9a235b92f672", "size": 507, "artifact_path": "wandb-client-artifact://2m8jrh8qy1kjp15i768gd9bhjsgyegx655d82aglxfydktnv8e844rrx42rrsknaa9pbyrawpm4e8qs9xn84rfaiuc5nkizt49dxy1z9q5wnt0liqqb2lxhszm6xknnf/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9r436wuk6yrj412kcbe2h9mjjbslwvkmzbgddqcbj1k1vuj3mw7iipt7uhr4l3a752kpqd3choyqq3g4kumw5senb5i0bhb97keh5wf00cu1svy03cf7xczfhfek26jd:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 250}, "eval/total_success_rate": 0.3199999928474426, "eval/total_success_rate_lb_0.01": 0.13405999541282654, "eval/total_success_rate_ub_0.01": 0.5579540729522705, "eval/total_success_rate_lb_0.025": 0.16728098690509796, "eval/total_success_rate_ub_0.025": 0.5333203673362732, "eval/total_success_rate_lb_0.05": 0.18870286643505096, "eval/total_success_rate_ub_0.05": 0.47915592789649963, "eval/total_success_rate_lb_0.1": 0.20175445079803467, "eval/total_success_rate_ub_0.1": 0.4631612002849579} \ No newline at end of file diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug-internal.log b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46e5992b38e95739880019d6d6c763bfaca633c2 --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-21 21:35:52,330 INFO StreamThr :2653360 [internal.py:wandb_internal():85] W&B internal server running at pid: 2653360, started at: 2025-01-21 21:35:52.330081 +2025-01-21 21:35:52,331 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:35:52,346 INFO WriterThread:2653360 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb +2025-01-21 21:35:52,346 DEBUG SenderThread:2653360 [sender.py:send():379] send: header +2025-01-21 21:35:52,354 DEBUG SenderThread:2653360 [sender.py:send():379] send: run +2025-01-21 21:35:52,577 INFO SenderThread:2653360 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files +2025-01-21 21:35:52,577 INFO SenderThread:2653360 [sender.py:_start_run_threads():1188] run started: hcp6xxlt with start time 1737513352.330422 +2025-01-21 21:35:52,592 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:35:52,593 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:35:52,668 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:35:52,673 DEBUG HandlerThread:2653360 [system_info.py:__init__():26] System info init +2025-01-21 21:35:52,673 DEBUG HandlerThread:2653360 [system_info.py:__init__():41] System info init done +2025-01-21 21:35:52,673 INFO HandlerThread:2653360 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:35:52,673 INFO SystemMonitor:2653360 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:35:52,673 INFO HandlerThread:2653360 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:35:52,674 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:35:52,675 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:35:52,676 INFO SystemMonitor:2653360 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:35:52,687 DEBUG HandlerThread:2653360 [system_info.py:probe():152] Probing system +2025-01-21 21:35:52,689 DEBUG HandlerThread:2653360 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_info.py:probe():200] Probing system done +2025-01-21 21:35:52,693 DEBUG HandlerThread:2653360 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:35:52.687764', 'startedAt': '2025-01-22T02:35:52.322613', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.96425, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.806, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 392.2366714477539}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:35:52,694 INFO HandlerThread:2653360 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:35:52,694 INFO HandlerThread:2653360 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:35:52,694 DEBUG HandlerThread:2653360 [system_info.py:_save_code():44] Saving code +2025-01-21 21:35:52,695 DEBUG HandlerThread:2653360 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:35:52,695 DEBUG HandlerThread:2653360 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:35:52,759 DEBUG HandlerThread:2653360 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:35:52,760 INFO HandlerThread:2653360 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:35:52,763 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:35:52,763 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:35:52,763 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:35:52,764 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:35:53,001 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:35:53,002 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:35:53,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:53,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:53,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:53,005 INFO HandlerThread:2653360 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/tb" + save: true +} +control { + relay_id: "55dd0b2c1b1743ffb18214d1539a47a8" +} +_info { + stream_id: "hcp6xxlt" +} + +2025-01-21 21:35:53,005 DEBUG HandlerThread:2653360 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:35:53,022 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/njokud3o-code/FISH/eval_robot.py +2025-01-21 21:35:53,062 DEBUG SenderThread:2653360 [sender.py:send():379] send: telemetry +2025-01-21 21:35:53,062 DEBUG SenderThread:2653360 [sender.py:send():379] send: tbrecord +2025-01-21 21:35:53,063 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:35:53,063 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 with policy live +2025-01-21 21:35:53,068 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/raznx8jb-diff.patch +2025-01-21 21:35:53,133 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/qrzo0zai-wandb-metadata.json +2025-01-21 21:35:53,259 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/6hrrc4bp-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json +2025-01-21 21:35:53,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH +2025-01-21 21:35:53,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code +2025-01-21 21:35:54,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:55,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:55,578 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:56,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:56,324 DEBUG SenderThread:2653360 [sender.py:send():379] send: config +2025-01-21 21:35:57,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:57,335 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:57,579 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:35:58,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:59,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:59,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:00,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:01,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:01,580 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:02,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:03,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:03,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:03,581 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:04,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:05,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:05,582 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:06,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:07,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:07,582 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:08,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:08,002 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:08,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:08,091 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:09,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:10,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:11,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:11,583 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:12,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:13,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:14,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:14,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:15,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:16,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:17,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:18,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:19,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:19,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:20,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:21,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:22,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:23,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:23,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:23,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:24,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:24,280 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:24,507 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:36:24,507 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:36:24,508 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 with policy now +2025-01-21 21:36:24,508 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:36:24,508 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:36:24,509 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos +2025-01-21 21:36:24,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media +2025-01-21 21:36:24,774 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/oji3jzi9-media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:25,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:25,041 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/glpo002s-media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:36:25,587 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:26,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:27,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:28,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:29,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:29,588 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:30,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:30,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:31,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:31,589 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:36:32,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:33,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:33,590 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:34,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:35,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:35,196 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:36,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:37,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:37,592 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:38,002 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:38,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:38,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:39,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:40,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:41,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:41,088 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:42,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:43,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:44,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:45,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:45,595 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:46,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:46,225 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:47,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:48,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:49,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:50,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:51,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:51,251 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:51,597 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:52,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:52,548 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:36:52,549 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 with policy now +2025-01-21 21:36:52,551 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:36:52,552 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:36:52,552 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:36:52,552 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:52,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:36:52,677 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:36:52,677 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:36:52,792 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w01589gh-media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:53,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:36:53,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:36:53,013 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:53,056 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/tzmlapo8-media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:36:53,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:54,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:54,598 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:55,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:56,016 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:57,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:57,237 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:36:57,599 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:36:58,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:36:59,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:00,016 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:01,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:01,601 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:02,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:02,717 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:03,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:03,601 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:04,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:05,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:06,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:07,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:08,003 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:08,003 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:08,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:08,117 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:09,017 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:10,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:11,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:12,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:13,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:14,007 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:14,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:15,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:16,018 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:17,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:18,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:19,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:20,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:20,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:21,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:22,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:22,678 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:37:23,005 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:23,005 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:23,045 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:24,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:25,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:25,112 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:26,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:27,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:28,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:29,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:30,021 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:31,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:31,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:32,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:33,022 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:34,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:35,026 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:36,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:37,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:37,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:38,008 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:38,008 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:38,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:39,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:40,024 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:40,614 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:37:40,873 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/gdo8dt21-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:37:41,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:41,406 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:37:41,406 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:37:41,407 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:37:41,614 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:37:41,615 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:42,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:42,581 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:43,025 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:43,616 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:44,026 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:45,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:45,616 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:46,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:47,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:47,617 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:47,991 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:48,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:49,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:50,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:51,049 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:52,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:52,678 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:37:53,004 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:37:53,004 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:37:53,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:53,133 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:54,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:55,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:56,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:57,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:58,050 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:58,684 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:37:59,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:37:59,621 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:37:59,867 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:37:59,867 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 with policy now +2025-01-21 21:37:59,869 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:37:59,870 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:37:59,870 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:37:59,870 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:00,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:00,110 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/xjfdismv-media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:00,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:38:00,916 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/clyopkyv-media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:38:01,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:01,622 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:02,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:03,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:04,057 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:04,481 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:05,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:05,624 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:06,064 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:07,058 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:07,624 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:08,009 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:08,010 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:08,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:09,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:09,223 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:09,223 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:09,236 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:09,625 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:09,723 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:10,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:11,063 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:11,626 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:12,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:13,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:13,626 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:14,060 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:15,010 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:15,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:15,627 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:16,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:17,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:18,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:19,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:20,011 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:20,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:21,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:22,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:22,679 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:38:23,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:23,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:23,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:24,061 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:25,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:25,133 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:26,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:27,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:28,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:29,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:30,062 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:30,290 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:31,067 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:31,560 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:38:31,560 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 with policy now +2025-01-21 21:38:31,562 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:38:31,563 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:31,563 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:31,563 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:31,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:38:31,816 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/1hm85y0w-media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:32,065 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:32,089 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/bbyywmp8-media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:38:33,067 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:33,633 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:34,076 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:35,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:35,833 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:36,068 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:37,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:37,635 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:38,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:38,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:38,066 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:38,635 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:39,077 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:39,260 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:38:39,261 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:38:39,261 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:38:39,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:40,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:41,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:41,636 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:41,725 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:42,078 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:43,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:43,637 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:44,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:45,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:45,638 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:46,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:47,069 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:47,638 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:38:47,721 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:48,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:49,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:50,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:51,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:52,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:52,679 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:38:53,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:38:53,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:38:53,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:53,126 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:54,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:55,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:56,070 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:57,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:58,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:38:59,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:38:59,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:00,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:01,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:02,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:03,071 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:04,019 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:04,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:05,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:06,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:07,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:08,006 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:08,006 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:08,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:09,072 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:09,124 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:10,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:11,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:12,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:13,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:14,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:15,020 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:15,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:16,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:17,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:18,073 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:19,074 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:20,098 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:20,379 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:20,458 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:20,459 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 with policy now +2025-01-21 21:39:20,465 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:20,465 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:20,465 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:20,470 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:20,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval +2025-01-21 21:39:20,728 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ch2imcub-media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:21,010 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w5du036o-media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:39:21,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:21,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:22,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:22,650 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:22,680 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:39:23,027 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:23,027 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:23,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:24,089 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:25,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:25,651 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:26,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:26,145 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:27,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:28,100 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:29,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:29,334 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:29,334 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:29,334 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:29,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:29,659 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:39:29,659 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:39:29,893 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:29,909 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/a22src9i-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:30,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:30,255 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:39:30,267 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:39:30,275 INFO wandb-upload_2:2653360 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9cbjc38d +2025-01-21 21:39:30,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:39:30,653 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table +2025-01-21 21:39:30,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval +2025-01-21 21:39:30,734 INFO SenderThread:2653360 [sender.py:send_request_log_artifact():1525] logged artifact run-hcp6xxlt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4NDA2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5ODM3MA==', 'latestArtifact': None}} +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:30,734 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:30,734 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,735 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send():379] send: telemetry +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,736 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,736 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,736 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,737 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,737 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,737 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,738 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,738 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,738 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,739 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,739 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,739 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,740 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,740 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,740 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,740 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:30,741 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:30,741 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:39:30,917 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ijswz3il-media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:30,927 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/d5y6if5t-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:31,092 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:31,111 INFO wandb-upload_2:2653360 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5vnbppk7 +2025-01-21 21:39:31,216 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/37wzxwli-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:39:31,515 INFO SenderThread:2653360 [sender.py:send_request_log_artifact():1525] logged artifact run-hcp6xxlt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY4NDA4Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5ODM3Mw==', 'latestArtifact': None}} +2025-01-21 21:39:31,515 DEBUG SenderThread:2653360 [sender.py:send():379] send: files +2025-01-21 21:39:31,515 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json with policy now +2025-01-21 21:39:31,516 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:31,516 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:39:31,516 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:39:31,517 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:39:31,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:39:31,654 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:39:31,788 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/w18vmkmu-media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:39:32,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:33,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:34,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:35,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:36,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:37,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:37,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:38,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:38,012 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:38,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:39,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:40,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:41,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:42,093 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:42,122 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:42,658 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:39:43,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:44,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:45,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:46,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:47,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:48,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:48,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:49,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:50,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:51,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:52,094 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:52,680 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:39:53,012 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:39:53,012 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:39:53,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:53,153 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:54,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:55,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:56,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:57,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:58,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:39:59,023 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:39:59,095 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:00,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:01,096 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:02,097 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:03,097 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:03,521 DEBUG SenderThread:2653360 [sender.py:send():379] send: exit +2025-01-21 21:40:03,521 INFO SenderThread:2653360 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:40:03,522 INFO SenderThread:2653360 [sender.py:send_exit():588] handling runtime: 250 +2025-01-21 21:40:03,522 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:send_exit():594] send defer +2025-01-21 21:40:03,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,523 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:40:03,523 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:40:03,523 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:40:03,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,523 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:40:03,524 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,524 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:40:03,524 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:40:03,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:40:03,524 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:40:03,524 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:40:03,525 DEBUG SystemMonitor:2653360 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:40:03,579 INFO HandlerThread:2653360 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:40:03,579 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,579 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:40:03,579 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:40:03,580 DEBUG SenderThread:2653360 [sender.py:send():379] send: stats +2025-01-21 21:40:03,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,580 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:40:03,580 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:03,580 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:40:03,580 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:40:03,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:03,580 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:40:03,668 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:05,670 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,343 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:40:10,344 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,345 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:40:10,345 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send():379] send: history +2025-01-21 21:40:10,345 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:40:10,345 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,346 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:40:10,346 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:10,346 DEBUG SenderThread:2653360 [sender.py:send():379] send: summary +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:40:10,347 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:40:10,347 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,347 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:40:10,347 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:40:10,347 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:40:10,347 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:40:10,348 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:10,348 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:40:10,348 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:10,348 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:40:10,523 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:10,672 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:12,601 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:40:12,601 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:12,601 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:12,602 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:40:12,602 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:12,602 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:40:12,602 INFO SenderThread:2653360 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:40:12,604 INFO SenderThread:2653360 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:40:12,606 INFO SenderThread:2653360 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:40:12,621 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:40:12,621 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:12,621 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:40:12,621 DEBUG SenderThread:2653360 [sender.py:send():379] send: artifact +2025-01-21 21:40:12,673 INFO Thread-12 :2653360 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxOTQ3OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:40:12,817 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:40:12,817 INFO SenderThread:2653360 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:40:13,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:13,673 INFO SenderThread:2653360 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files +2025-01-21 21:40:13,673 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml config.yaml +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json wandb-summary.json +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt requirements.txt +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log output.log +2025-01-21 21:40:13,674 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/diff.patch diff.patch +2025-01-21 21:40:13,679 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 media/videos/eval/0_eval_0_b5a262079efd287fc775.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 media/videos/eval/1_eval_1_a81d19fe2171d168a019.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 media/videos/eval/2_eval_2_7f4db46f72773c3d5eca.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 media/videos/eval/3_eval_3_99b67859ddc474a3c0ab.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 media/videos/eval/4_eval_4_d14611aa0de1dd53b110.mp4 +2025-01-21 21:40:13,682 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json media/table/eval/total_success_rate_ci_6_de2f891a47c3da512984.table.json +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:40:13,683 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:13,683 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:13,683 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:40:13,683 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:40:13,683 INFO SenderThread:2653360 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:40:13,862 INFO wandb-upload_2:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/wandb-summary.json +2025-01-21 21:40:13,865 INFO wandb-upload_0:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/requirements.txt +2025-01-21 21:40:13,950 INFO wandb-upload_4:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/output.log +2025-01-21 21:40:13,955 INFO wandb-upload_1:2653360 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/files/config.yaml +2025-01-21 21:40:13,996 INFO wandb-upload_3:2653360 [upload_job.py:push():130] Uploaded file /tmp/tmp075rztjowandb/ee06zo97-events.out.tfevents.1737513353.leonmkim-ROG-Strix-G15CS-G15CS.2653170.0 +2025-01-21 21:40:14,196 INFO Thread-11 (_thread_body):2653360 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:40:14,197 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,197 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:40:14,197 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,197 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:40:14,197 INFO SenderThread:2653360 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:40:14,198 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:40:14,198 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,198 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:40:14,199 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,199 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:40:14,199 INFO SenderThread:2653360 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:40:14,524 INFO SenderThread:2653360 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:40:14,524 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:40:14,524 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,525 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:40:14,525 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,525 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,525 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:40:14,525 INFO SenderThread:2653360 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:40:14,526 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:40:14,526 INFO HandlerThread:2653360 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send():379] send: final +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send():379] send: footer +2025-01-21 21:40:14,526 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: defer +2025-01-21 21:40:14,526 INFO SenderThread:2653360 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:40:14,527 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,527 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:40:14,528 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:40:14,528 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:40:14,530 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:40:14,579 DEBUG SenderThread:2653360 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:40:14,579 INFO MainThread:2653360 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:40:14,580 INFO MainThread:2653360 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:40:14,580 INFO MainThread:2653360 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:40:14,580 DEBUG HandlerThread:2653360 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:40:14,580 INFO HandlerThread:2653360 [handler.py:finish():882] shutting down handler +2025-01-21 21:40:15,528 INFO WriterThread:2653360 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [sender.py:finish():1615] shutting down sender +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:40:15,579 INFO SenderThread:2653360 [file_pusher.py:join():175] waiting for file pusher diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug.log b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d852ab8f745eaad1ac7907eebbc80ae5819c6cdf --- /dev/null +++ b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Configure stats pid to 2653170 +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/settings +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug.log +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/logs/debug-internal.log +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt'} +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():619] starting backend +2025-01-21 21:35:52,324 INFO MainThread:2653170 [wandb_init.py:init():623] setting up manager +2025-01-21 21:35:52,329 INFO MainThread:2653170 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:35:52,330 INFO MainThread:2653170 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:35:52,345 INFO MainThread:2653170 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:35:52,351 INFO MainThread:2653170 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:35:52,592 INFO MainThread:2653170 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:35:52,661 INFO MainThread:2653170 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:35:52,661 INFO MainThread:2653170 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:35:53,002 INFO MainThread:2653170 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:35:53,004 INFO MainThread:2653170 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:35:53,005 INFO MainThread:2653170 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/hcp6xxlt/tb, True +2025-01-21 21:35:56,323 INFO MainThread:2653170 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:40:15,581 WARNING MsgRouterThr:2653170 [router.py:message_loop():77] message_loop has been closed diff --git a/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e5be8ed2c3dc265e210d178a98dc6f808e29cb3f Binary files /dev/null and b/hcp6xxlt/wandb/run-20250121_213552-hcp6xxlt/run-hcp6xxlt.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0a7a96935f93a39607e978391682f9f2933758b --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5a3e28e9e7e6478ed830d39acc2c6665f8d408268e191366db69ebb7260fd4 +size 910332870 diff --git a/snapshot_12499.pt b/snapshot_12499.pt new file mode 100644 index 0000000000000000000000000000000000000000..899db98f8bdc44855cd29e433fb6812355c4819f --- /dev/null +++ b/snapshot_12499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:501116078d0c0911266f3e91c33779b9b26c7e3baadfa3bf5c4331d6480ba79b +size 910332870 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..bee78a6999a843f3de94c9b75af3438a18458544 --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c46480e58f765b1133b7869c69a9a6ba50374b5cdbfeaf0a4c5955a41d3b6891 +size 910332870 diff --git a/snapshot_13499.pt b/snapshot_13499.pt new file mode 100644 index 0000000000000000000000000000000000000000..61ec7138c16a25170f3bcbc49ee3d2cf38a8b695 --- /dev/null +++ b/snapshot_13499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15fbda103a8c2093f8171ab83db342b9c924c4f67a621b75635525e2f6bf635 +size 910332870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb3735438ca7d6907c395a20d94e28af5813de37 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71e4d32517d0982c612a8c5400e4e6e2cdad9e27762581238ad506d099499e60 +size 910332870 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7954e3ebe56ece3ba9379331c861d63edb2ce6cd --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19230d06b63c228ab6683113981a7ca42a4892e229365211695d0c4add9391f +size 910332870 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6e2af65a91719d5349330be206dd7ac4340df93 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce00a93c3cec2e19da9e5373004abcc7208f7c1a534a1fef18006fce1f5d6b1b +size 910332360 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfb83a0553e1c1cc610b8884261829d2584fd15b --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5133b3f8c0003d299317098f1520644170617dcdf3dc1e3c69dd1c39db039aaf +size 910332360 diff --git a/snapshot_6749.pt b/snapshot_6749.pt new file mode 100644 index 0000000000000000000000000000000000000000..542a6bf15cd310c80ce8779215d97b3be1e06472 --- /dev/null +++ b/snapshot_6749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee493443388d38100803e95b48fc67e5aba21581f0802ea5950f02d639fba8e +size 910332360 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..507e501beb25ca7e38fb61730bb5792fa815b3e8 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff513229c923eb16c1ea93d6add81502735db445d21678d059829ebf160e9167 +size 910332360 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d102cadc7f4d911eec73e2e561b9cee7f7e026a5 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3953dfb3fae90a17acb1e29d3bcd88bba195ce343cccd4338f6bfd775c79706 +size 910332870 diff --git a/snapshot_9499.pt b/snapshot_9499.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd31b2a2559dc009cef0806dcde42dc1b98d68a --- /dev/null +++ b/snapshot_9499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c9abfed0f1f2f4e5d2580807cb3cca59bab525f9ef24a84bba062aae6ec58e +size 910332870 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..7954e3ebe56ece3ba9379331c861d63edb2ce6cd --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19230d06b63c228ab6683113981a7ca42a4892e229365211695d0c4add9391f +size 910332870 diff --git a/zdpaop5m/.hydra/config.yaml b/zdpaop5m/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90279a1d8d4077f1affa573bd8d5db0ff727e458 --- /dev/null +++ b/zdpaop5m/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/zdpaop5m/.hydra/hydra.yaml b/zdpaop5m/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88994f32e4fee3ffa1d2a2b542d21d79e1918a60 --- /dev/null +++ b/zdpaop5m/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/zdpaop5m/.hydra/overrides.yaml b/zdpaop5m/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/zdpaop5m/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/zdpaop5m/episode_rosbags/aligned_depth_to_color_K.npy b/zdpaop5m/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zdpaop5m/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zdpaop5m/episode_rosbags/cam_tf_world.npy b/zdpaop5m/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/zdpaop5m/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/zdpaop5m/episode_rosbags/color_K.npy b/zdpaop5m/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/zdpaop5m/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/zdpaop5m/episode_rosbags/depth_K.npy b/zdpaop5m/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/zdpaop5m/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/zdpaop5m/eval_robot.log b/zdpaop5m/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b6b95016c360bce1d758a45c84506cdd34d288a2 --- /dev/null +++ b/zdpaop5m/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 20:41:09,512][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:41:09,515][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/zdpaop5m/tb/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 b/zdpaop5m/tb/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 new file mode 100644 index 0000000000000000000000000000000000000000..f0f64d2525b88c52b23eabf36f908e189028c3fd --- /dev/null +++ b/zdpaop5m/tb/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66cdc7a864518752e9e2103f56653866b4eac7d816c32d50df6dbdd2d092a69b +size 88 diff --git a/zdpaop5m/wandb/debug-internal.log b/zdpaop5m/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..26f8cd07667460aa1ebc3fe0f939af23abcdebd5 --- /dev/null +++ b/zdpaop5m/wandb/debug-internal.log @@ -0,0 +1,238 @@ +2025-01-21 20:41:15,462 INFO StreamThr :2638850 [internal.py:wandb_internal():85] W&B internal server running at pid: 2638850, started at: 2025-01-21 20:41:15.462434 +2025-01-21 20:41:15,463 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:41:15,474 INFO WriterThread:2638850 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb +2025-01-21 20:41:15,475 DEBUG SenderThread:2638850 [sender.py:send():379] send: header +2025-01-21 20:41:15,484 DEBUG SenderThread:2638850 [sender.py:send():379] send: run +2025-01-21 20:41:15,824 INFO SenderThread:2638850 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files +2025-01-21 20:41:15,824 INFO SenderThread:2638850 [sender.py:_start_run_threads():1188] run started: zdpaop5m with start time 1737510075.463003 +2025-01-21 20:41:15,839 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:41:15,839 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:41:15,888 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:41:15,893 DEBUG HandlerThread:2638850 [system_info.py:__init__():26] System info init +2025-01-21 20:41:15,893 DEBUG HandlerThread:2638850 [system_info.py:__init__():41] System info init done +2025-01-21 20:41:15,893 INFO HandlerThread:2638850 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:41:15,893 INFO SystemMonitor:2638850 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:41:15,894 INFO HandlerThread:2638850 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:41:15,894 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:41:15,895 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:41:15,895 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:41:15,896 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:41:15,896 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:41:15,909 DEBUG HandlerThread:2638850 [system_info.py:probe():152] Probing system +2025-01-21 20:41:15,910 DEBUG HandlerThread:2638850 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:probe():200] Probing system done +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:41:15.909487', 'startedAt': '2025-01-22T01:41:15.454898', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.03515243530273}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:41:15,916 INFO HandlerThread:2638850 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:41:15,916 INFO HandlerThread:2638850 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:_save_code():44] Saving code +2025-01-21 20:41:15,919 DEBUG HandlerThread:2638850 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:41:15,919 DEBUG HandlerThread:2638850 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:41:15,950 DEBUG HandlerThread:2638850 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:41:15,950 INFO HandlerThread:2638850 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:41:15,953 DEBUG SenderThread:2638850 [sender.py:send():379] send: files +2025-01-21 20:41:15,953 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:41:15,954 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:41:16,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:41:16,184 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:41:16,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:41:16,184 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:41:16,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:16,188 INFO HandlerThread:2638850 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/tb" + save: true +} +control { + relay_id: "47ddae32cc3b49d296447496c5dc89e7" +} +_info { + stream_id: "zdpaop5m" +} + +2025-01-21 20:41:16,188 DEBUG HandlerThread:2638850 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:41:16,245 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/zdcuvt79-code/FISH/eval_robot.py +2025-01-21 20:41:16,329 INFO wandb-upload_0:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/zxnnlzs3-wandb-metadata.json +2025-01-21 20:41:16,332 DEBUG SenderThread:2638850 [sender.py:send():379] send: telemetry +2025-01-21 20:41:16,332 DEBUG SenderThread:2638850 [sender.py:send():379] send: tbrecord +2025-01-21 20:41:16,333 DEBUG SenderThread:2638850 [sender.py:send():379] send: files +2025-01-21 20:41:16,333 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 with policy live +2025-01-21 20:41:16,526 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/3givq3co-events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH +2025-01-21 20:41:17,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:18,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:18,825 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:19,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:20,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:21,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:21,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:22,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:23,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:24,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:25,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:26,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:26,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:27,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:28,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:29,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:30,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:31,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:41:31,186 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:41:31,187 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:31,286 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:31,288 DEBUG SenderThread:2638850 [sender.py:send():379] send: exit +2025-01-21 20:41:31,288 INFO SenderThread:2638850 [sender.py:send_exit():586] handling exit code: 1 +2025-01-21 20:41:31,288 INFO SenderThread:2638850 [sender.py:send_exit():588] handling runtime: 15 +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:send_exit():594] send defer +2025-01-21 20:41:31,289 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,289 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 20:41:31,289 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 1 +2025-01-21 20:41:31,289 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,289 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 20:41:31,290 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,290 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 20:41:31,290 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 2 +2025-01-21 20:41:31,290 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 20:41:31,290 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:41:31,290 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 20:41:31,291 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 20:41:31,291 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 20:41:31,346 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 20:41:31,347 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 20:41:31,347 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined network monitor +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 3 +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send():379] send: stats +2025-01-21 20:41:31,348 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,348 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 4 +2025-01-21 20:41:31,348 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,348 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 20:41:31,829 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:32,829 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,192 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 20:41:38,192 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 5 +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:38,192 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,192 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send():379] send: summary +2025-01-21 20:41:38,193 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:41:38,193 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,193 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 20:41:38,194 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 6 +2025-01-21 20:41:38,194 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,194 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 20:41:38,194 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,194 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 20:41:38,197 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:38,313 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:38,317 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 7 +2025-01-21 20:41:38,317 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:38,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,317 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 20:41:38,317 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,317 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 8 +2025-01-21 20:41:38,752 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,752 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 20:41:38,752 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 20:41:38,754 INFO SenderThread:2638850 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 20:41:38,773 INFO SenderThread:2638850 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 20:41:38,780 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 9 +2025-01-21 20:41:38,781 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,781 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 20:41:38,781 DEBUG SenderThread:2638850 [sender.py:send():379] send: artifact +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:39,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:39,567 INFO wandb-upload_0:2638850 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwcdc1cck +2025-01-21 20:41:39,809 INFO wandb-upload_1:2638850 [upload_job.py:push():88] Uploaded file /tmp/tmpct2tyczq/wandb-job.json +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxNDI3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTk0MA==', 'versionIndex': 451}}} +2025-01-21 20:41:40,422 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 20:41:40,831 INFO SenderThread:2638850 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml config.yaml +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json wandb-summary.json +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt requirements.txt +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json wandb-metadata.json +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log output.log +2025-01-21 20:41:40,838 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 20:41:40,838 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 10 +2025-01-21 20:41:40,838 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:40,839 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:40,839 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 20:41:40,839 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:40,839 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 20:41:40,839 INFO SenderThread:2638850 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:41:41,014 INFO wandb-upload_0:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:41,024 INFO wandb-upload_2:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml +2025-01-21 20:41:41,062 INFO wandb-upload_3:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt +2025-01-21 20:41:41,077 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:41,278 INFO Thread-11 (_thread_body):2638850 [sender.py:transition_state():614] send defer: 11 +2025-01-21 20:41:41,278 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,278 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 20:41:41,278 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,278 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 20:41:41,278 INFO SenderThread:2638850 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 12 +2025-01-21 20:41:41,279 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,279 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 20:41:41,279 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [file_stream.py:finish():601] file stream finish called +2025-01-21 20:41:41,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [file_stream.py:finish():605] file stream finish is done +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 13 +2025-01-21 20:41:41,366 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,366 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,366 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 20:41:41,366 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 14 +2025-01-21 20:41:41,367 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,367 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send():379] send: final +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send():379] send: footer +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,367 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 20:41:41,368 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,368 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 20:41:41,368 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,368 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: server_info +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:41,420 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 20:41:41,421 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 20:41:41,421 INFO HandlerThread:2638850 [handler.py:finish():882] shutting down handler +2025-01-21 20:41:42,370 INFO WriterThread:2638850 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [sender.py:finish():1615] shutting down sender +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [file_pusher.py:join():175] waiting for file pusher diff --git a/zdpaop5m/wandb/debug.log b/zdpaop5m/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..019dc8b79d396b966e245fc9ca1765ac621bbcd6 --- /dev/null +++ b/zdpaop5m/wandb/debug.log @@ -0,0 +1,30 @@ +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Configure stats pid to 2638664 +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/settings +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug.log +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug-internal.log +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m'} +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():619] starting backend +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():623] setting up manager +2025-01-21 20:41:15,461 INFO MainThread:2638664 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:41:15,462 INFO MainThread:2638664 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:41:15,474 INFO MainThread:2638664 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:41:15,482 INFO MainThread:2638664 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:41:15,838 INFO MainThread:2638664 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:41:15,883 INFO MainThread:2638664 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:41:15,883 INFO MainThread:2638664 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:41:16,187 INFO MainThread:2638664 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:41:16,188 INFO MainThread:2638664 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/tb, True +2025-01-21 20:41:42,421 WARNING MsgRouterThr:2638664 [router.py:message_loop():77] message_loop has been closed diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1228943b98822d4ea8cc00c155559a55bace066 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml @@ -0,0 +1,929 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_0' +wandb_notes: + desc: null + value: 4295_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737510075 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..06ee1c215e4dfd53bcfd5d9f989ac77428655d7d --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log @@ -0,0 +1,15 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +program exiting gracefully +Error executing job with overrides: ['agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'] +Traceback (most recent call last): + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 603, in main + workspace.eval() + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 230, in eval + left_book_slot = input("Enter the left book slot location: ") + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/core.py", line 610, in _ros_signal + prev_handler(sig, stackframe) + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 142, in signal_handler + self.proceed_after_env_reset_event.set() +AttributeError: 'Workspace' object has no attribute 'proceed_after_env_reset_event' +Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace. \ No newline at end of file diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d465424d4bdb242e95f3f5981acaa7a1cea318c0 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T01:41:15.909487", + "startedAt": "2025-01-22T01:41:15.454898", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.021, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.024, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.019, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 311.03515243530273 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5118b50d79d9370dee544d0cb19c11385cc1bdf0 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 15}} \ No newline at end of file diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug-internal.log b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..26f8cd07667460aa1ebc3fe0f939af23abcdebd5 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug-internal.log @@ -0,0 +1,238 @@ +2025-01-21 20:41:15,462 INFO StreamThr :2638850 [internal.py:wandb_internal():85] W&B internal server running at pid: 2638850, started at: 2025-01-21 20:41:15.462434 +2025-01-21 20:41:15,463 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status +2025-01-21 20:41:15,474 INFO WriterThread:2638850 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb +2025-01-21 20:41:15,475 DEBUG SenderThread:2638850 [sender.py:send():379] send: header +2025-01-21 20:41:15,484 DEBUG SenderThread:2638850 [sender.py:send():379] send: run +2025-01-21 20:41:15,824 INFO SenderThread:2638850 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files +2025-01-21 20:41:15,824 INFO SenderThread:2638850 [sender.py:_start_run_threads():1188] run started: zdpaop5m with start time 1737510075.463003 +2025-01-21 20:41:15,839 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 20:41:15,839 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: check_version +2025-01-21 20:41:15,888 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 20:41:15,893 DEBUG HandlerThread:2638850 [system_info.py:__init__():26] System info init +2025-01-21 20:41:15,893 DEBUG HandlerThread:2638850 [system_info.py:__init__():41] System info init done +2025-01-21 20:41:15,893 INFO HandlerThread:2638850 [system_monitor.py:start():194] Starting system monitor +2025-01-21 20:41:15,893 INFO SystemMonitor:2638850 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 20:41:15,894 INFO HandlerThread:2638850 [system_monitor.py:probe():214] Collecting system info +2025-01-21 20:41:15,894 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 20:41:15,895 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started disk monitoring +2025-01-21 20:41:15,895 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 20:41:15,896 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started memory monitoring +2025-01-21 20:41:15,896 INFO SystemMonitor:2638850 [interfaces.py:start():188] Started network monitoring +2025-01-21 20:41:15,909 DEBUG HandlerThread:2638850 [system_info.py:probe():152] Probing system +2025-01-21 20:41:15,910 DEBUG HandlerThread:2638850 [system_info.py:_probe_git():137] Probing git +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:_probe_git():145] Probing git done +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:probe():200] Probing system done +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T01:41:15.909487', 'startedAt': '2025-01-22T01:41:15.454898', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.03515243530273}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 20:41:15,916 INFO HandlerThread:2638850 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 20:41:15,916 INFO HandlerThread:2638850 [system_monitor.py:probe():227] Publishing system info +2025-01-21 20:41:15,916 DEBUG HandlerThread:2638850 [system_info.py:_save_code():44] Saving code +2025-01-21 20:41:15,919 DEBUG HandlerThread:2638850 [system_info.py:_save_code():67] Saving code done +2025-01-21 20:41:15,919 DEBUG HandlerThread:2638850 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 20:41:15,950 DEBUG HandlerThread:2638850 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 20:41:15,950 INFO HandlerThread:2638850 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 20:41:15,953 DEBUG SenderThread:2638850 [sender.py:send():379] send: files +2025-01-21 20:41:15,953 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 20:41:15,954 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 20:41:16,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 20:41:16,184 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: python_packages +2025-01-21 20:41:16,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:41:16,184 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:41:16,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:16,188 INFO HandlerThread:2638850 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/tb" + save: true +} +control { + relay_id: "47ddae32cc3b49d296447496c5dc89e7" +} +_info { + stream_id: "zdpaop5m" +} + +2025-01-21 20:41:16,188 DEBUG HandlerThread:2638850 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 20:41:16,245 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/zdcuvt79-code/FISH/eval_robot.py +2025-01-21 20:41:16,329 INFO wandb-upload_0:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/zxnnlzs3-wandb-metadata.json +2025-01-21 20:41:16,332 DEBUG SenderThread:2638850 [sender.py:send():379] send: telemetry +2025-01-21 20:41:16,332 DEBUG SenderThread:2638850 [sender.py:send():379] send: tbrecord +2025-01-21 20:41:16,333 DEBUG SenderThread:2638850 [sender.py:send():379] send: files +2025-01-21 20:41:16,333 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 with policy live +2025-01-21 20:41:16,526 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /tmp/tmpuc_ptyqswandb/3givq3co-events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt +2025-01-21 20:41:16,826 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code +2025-01-21 20:41:16,827 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH +2025-01-21 20:41:17,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:18,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:18,825 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:19,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:20,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:21,184 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:21,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:22,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:23,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:24,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:25,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:26,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:26,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:27,185 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:28,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:29,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:30,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:31,186 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 20:41:31,186 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: stop_status +2025-01-21 20:41:31,187 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:31,286 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:31,288 DEBUG SenderThread:2638850 [sender.py:send():379] send: exit +2025-01-21 20:41:31,288 INFO SenderThread:2638850 [sender.py:send_exit():586] handling exit code: 1 +2025-01-21 20:41:31,288 INFO SenderThread:2638850 [sender.py:send_exit():588] handling runtime: 15 +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:send_exit():594] send defer +2025-01-21 20:41:31,289 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,289 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 20:41:31,289 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 20:41:31,289 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 1 +2025-01-21 20:41:31,289 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,289 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 20:41:31,290 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,290 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 20:41:31,290 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 2 +2025-01-21 20:41:31,290 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 20:41:31,290 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 20:41:31,290 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 20:41:31,290 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 20:41:31,291 DEBUG SystemMonitor:2638850 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 20:41:31,291 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 20:41:31,346 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 20:41:31,347 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 20:41:31,347 INFO HandlerThread:2638850 [interfaces.py:finish():200] Joined network monitor +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 3 +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send():379] send: stats +2025-01-21 20:41:31,348 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,348 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 20:41:31,348 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 20:41:31,348 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 4 +2025-01-21 20:41:31,348 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:31,348 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 20:41:31,829 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:32,829 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:38,191 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,192 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 20:41:38,192 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 5 +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:38,192 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,192 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 20:41:38,192 DEBUG SenderThread:2638850 [sender.py:send():379] send: summary +2025-01-21 20:41:38,193 INFO SenderThread:2638850 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 20:41:38,193 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,193 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 20:41:38,194 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 6 +2025-01-21 20:41:38,194 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,194 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 20:41:38,194 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,194 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 20:41:38,197 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 20:41:38,313 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:38,317 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 7 +2025-01-21 20:41:38,317 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:38,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,317 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 20:41:38,317 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,317 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 8 +2025-01-21 20:41:38,752 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,752 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 20:41:38,752 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 20:41:38,752 INFO SenderThread:2638850 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 20:41:38,754 INFO SenderThread:2638850 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 20:41:38,773 INFO SenderThread:2638850 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 20:41:38,780 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 9 +2025-01-21 20:41:38,781 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:38,781 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 20:41:38,781 DEBUG SenderThread:2638850 [sender.py:send():379] send: artifact +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:38,831 INFO Thread-12 :2638850 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:39,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:39,567 INFO wandb-upload_0:2638850 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwcdc1cck +2025-01-21 20:41:39,809 INFO wandb-upload_1:2638850 [upload_job.py:push():88] Uploaded file /tmp/tmpct2tyczq/wandb-job.json +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYxNDI3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MjA0NTk0MA==', 'versionIndex': 451}}} +2025-01-21 20:41:40,422 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 20:41:40,422 INFO SenderThread:2638850 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 20:41:40,831 INFO SenderThread:2638850 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml config.yaml +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json wandb-summary.json +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt requirements.txt +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-metadata.json wandb-metadata.json +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 events.out.tfevents.1737510076.leonmkim-ROG-Strix-G15CS-G15CS.2638664.0 +2025-01-21 20:41:40,832 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log output.log +2025-01-21 20:41:40,838 INFO SenderThread:2638850 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 20:41:40,838 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 10 +2025-01-21 20:41:40,838 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:40,839 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:40,839 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 20:41:40,839 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:40,839 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 20:41:40,839 INFO SenderThread:2638850 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:41:41,014 INFO wandb-upload_0:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/wandb-summary.json +2025-01-21 20:41:41,024 INFO wandb-upload_2:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/config.yaml +2025-01-21 20:41:41,062 INFO wandb-upload_3:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/requirements.txt +2025-01-21 20:41:41,077 INFO wandb-upload_1:2638850 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/files/output.log +2025-01-21 20:41:41,278 INFO Thread-11 (_thread_body):2638850 [sender.py:transition_state():614] send defer: 11 +2025-01-21 20:41:41,278 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,278 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 20:41:41,278 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,278 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 20:41:41,278 INFO SenderThread:2638850 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 12 +2025-01-21 20:41:41,279 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,279 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 20:41:41,279 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 20:41:41,279 INFO SenderThread:2638850 [file_stream.py:finish():601] file stream finish called +2025-01-21 20:41:41,317 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [file_stream.py:finish():605] file stream finish is done +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 13 +2025-01-21 20:41:41,366 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,366 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,366 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 20:41:41,366 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 20:41:41,366 INFO SenderThread:2638850 [sender.py:transition_state():614] send defer: 14 +2025-01-21 20:41:41,367 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: defer +2025-01-21 20:41:41,367 INFO HandlerThread:2638850 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send():379] send: final +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send():379] send: footer +2025-01-21 20:41:41,367 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: defer +2025-01-21 20:41:41,367 INFO SenderThread:2638850 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 20:41:41,368 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,368 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 20:41:41,368 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,368 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: server_info +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 20:41:41,370 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 20:41:41,420 DEBUG SenderThread:2638850 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 20:41:41,421 INFO MainThread:2638850 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 20:41:41,421 DEBUG HandlerThread:2638850 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 20:41:41,421 INFO HandlerThread:2638850 [handler.py:finish():882] shutting down handler +2025-01-21 20:41:42,370 INFO WriterThread:2638850 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [sender.py:finish():1615] shutting down sender +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 20:41:42,421 INFO SenderThread:2638850 [file_pusher.py:join():175] waiting for file pusher diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug.log b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..019dc8b79d396b966e245fc9ca1765ac621bbcd6 --- /dev/null +++ b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug.log @@ -0,0 +1,30 @@ +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Configure stats pid to 2638664 +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/settings +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 20:41:15,456 INFO MainThread:2638664 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug.log +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/logs/debug-internal.log +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():569] calling init triggers +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_0', 'wandb_notes': '4295_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m'} +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():619] starting backend +2025-01-21 20:41:15,457 INFO MainThread:2638664 [wandb_init.py:init():623] setting up manager +2025-01-21 20:41:15,461 INFO MainThread:2638664 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 20:41:15,462 INFO MainThread:2638664 [wandb_init.py:init():631] backend started and connected +2025-01-21 20:41:15,474 INFO MainThread:2638664 [wandb_init.py:init():720] updated telemetry +2025-01-21 20:41:15,482 INFO MainThread:2638664 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 20:41:15,838 INFO MainThread:2638664 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 20:41:15,883 INFO MainThread:2638664 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 20:41:15,883 INFO MainThread:2638664 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 20:41:16,185 INFO MainThread:2638664 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 20:41:16,187 INFO MainThread:2638664 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 20:41:16,188 INFO MainThread:2638664 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_0/zdpaop5m/tb, True +2025-01-21 20:41:42,421 WARNING MsgRouterThr:2638664 [router.py:message_loop():77] message_loop has been closed diff --git a/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..dbe419f19285ec850cfa20455cecbd799eeba930 Binary files /dev/null and b/zdpaop5m/wandb/run-20250121_204115-zdpaop5m/run-zdpaop5m.wandb differ