diff --git a/.gitattributes b/.gitattributes index b6a0d957d2463a15d88c79a1a191666d9eba7f44..e772426fc78499ce3dcb42ca257d3f6521fb0cac 100644 --- a/.gitattributes +++ b/.gitattributes @@ -74,3 +74,50 @@ nbjbj4mc/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text nbjbj4mc/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text nbjbj4mc/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text nbjbj4mc/episode_rosbags/episode_1_2025-01-03-21-07-23.bag filter=lfs diff=lfs merge=lfs -text +nibxxtai/episode_rosbags/episode_2_2025-01-03-20-48-40.bag filter=lfs diff=lfs merge=lfs -text +nibxxtai/episode_rosbags/episode_0_2025-01-03-20-46-55.bag filter=lfs diff=lfs merge=lfs -text +nibxxtai/episode_rosbags/episode_4_2025-01-03-20-51-04.bag filter=lfs diff=lfs merge=lfs -text +nibxxtai/episode_rosbags/episode_3_2025-01-03-20-50-04.bag filter=lfs diff=lfs merge=lfs -text +nibxxtai/episode_rosbags/episode_1_2025-01-03-20-47-48.bag filter=lfs diff=lfs merge=lfs -text +97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +97ao8n2y/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/episode_rosbags/episode_1_2025-01-03-23-06-10.bag filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/episode_rosbags/episode_0_2025-01-03-23-05-20.bag filter=lfs diff=lfs merge=lfs -text +97ao8n2y/episode_rosbags/episode_4_2025-01-04-21-47-32.bag filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/episode_rosbags/episode_2_2025-01-03-23-06-58.bag filter=lfs diff=lfs merge=lfs -text +br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +br14p9rb/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/episode_rosbags/episode_3_2025-01-03-23-07-55.bag filter=lfs diff=lfs merge=lfs -text +kdczwpup/episode_rosbags/episode_4_2025-01-03-20-56-24.bag filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/episode_rosbags/episode_3_2025-01-03-21-09-09.bag filter=lfs diff=lfs merge=lfs -text +97ao8n2y/episode_rosbags/episode_1_2025-01-04-21-44-44.bag filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/episode_rosbags/episode_0_2025-01-03-21-06-34.bag filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/episode_rosbags/episode_2_2025-01-03-21-08-18.bag filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/episode_rosbags/episode_4_2025-01-03-23-08-49.bag filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/episode_rosbags/episode_4_2025-01-03-21-10-31.bag filter=lfs diff=lfs merge=lfs -text +75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +75aq4eke/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/75aq4eke/.hydra/config.yaml b/75aq4eke/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/75aq4eke/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/75aq4eke/.hydra/hydra.yaml b/75aq4eke/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..007a80b6794b18d46ffca05783f241c0f6685891 --- /dev/null +++ b/75aq4eke/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/75aq4eke/.hydra/overrides.yaml b/75aq4eke/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/75aq4eke/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/75aq4eke/episode_rosbags/aligned_depth_to_color_K.npy b/75aq4eke/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/75aq4eke/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/75aq4eke/episode_rosbags/cam_tf_world.npy b/75aq4eke/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/75aq4eke/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/75aq4eke/episode_rosbags/color_K.npy b/75aq4eke/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/75aq4eke/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/75aq4eke/episode_rosbags/depth_K.npy b/75aq4eke/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/75aq4eke/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/75aq4eke/eval_robot.log b/75aq4eke/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6e56977d5152253d6908482083a1f71115f58ae6 --- /dev/null +++ b/75aq4eke/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:59:19,583][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:59:19,586][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/75aq4eke/eval_video/0_eval.mp4 b/75aq4eke/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9253a06b2fecac86fc9e06dfb8d4c0b6c91dd2a0 --- /dev/null +++ b/75aq4eke/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a512bbba28a2d223fb5f72c7ce2558a5d852f6f2ec72797975b5240755d770a7 +size 1450027 diff --git a/75aq4eke/eval_video/1_eval.mp4 b/75aq4eke/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53edf35457f9050e0bb38d7b1b16d937a7aaec4f --- /dev/null +++ b/75aq4eke/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:320233e3aec19848df182bd08223f682721c3af9643e5ea1b17d6a45d1424e37 +size 1459964 diff --git a/75aq4eke/eval_video/2_eval.mp4 b/75aq4eke/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcd377d17a1d7154570038ad46d67f770a22ac18 --- /dev/null +++ b/75aq4eke/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1136994690265702739ef6163f5157efcef3fbe2b227a943e8f2cdf2c2a2d365 +size 1418203 diff --git a/75aq4eke/eval_video/3_eval.mp4 b/75aq4eke/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14f70accc2614c0b8a17163317e95cfd46245ca4 --- /dev/null +++ b/75aq4eke/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036663fd181fb684915fdf998c256bd7355bd26df88fa0cfd2f7e4cc9fdfdaa9 +size 977247 diff --git a/75aq4eke/eval_video/4_eval.mp4 b/75aq4eke/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e403881bfc0ee66dfc602113166a5fd33c931a8 --- /dev/null +++ b/75aq4eke/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a344ae7a6fd7d295455849ef0845435731ab099787293c1f6421d6969325b54 +size 1546270 diff --git a/75aq4eke/tb/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 b/75aq4eke/tb/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 new file mode 100644 index 0000000000000000000000000000000000000000..bdc0043eb598b70036b2b15796df0f30bb5f5854 --- /dev/null +++ b/75aq4eke/tb/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4280a0e9b230d1451f12e10d33c448c396ca944f63e9b90e39e4e0f853261197 +size 6539 diff --git a/75aq4eke/wandb/debug-internal.log b/75aq4eke/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f5dad30a043a8d044fe8751ef0993621433c61ba --- /dev/null +++ b/75aq4eke/wandb/debug-internal.log @@ -0,0 +1,879 @@ +2025-01-03 20:59:25,356 INFO StreamThr :1971983 [internal.py:wandb_internal():85] W&B internal server running at pid: 1971983, started at: 2025-01-03 20:59:25.355968 +2025-01-03 20:59:25,357 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:59:25,370 INFO WriterThread:1971983 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb +2025-01-03 20:59:25,370 DEBUG SenderThread:1971983 [sender.py:send():379] send: header +2025-01-03 20:59:25,378 DEBUG SenderThread:1971983 [sender.py:send():379] send: run +2025-01-03 20:59:25,583 INFO SenderThread:1971983 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files +2025-01-03 20:59:25,583 INFO SenderThread:1971983 [sender.py:_start_run_threads():1188] run started: 75aq4eke with start time 1735955965.356425 +2025-01-03 20:59:25,599 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:59:25,599 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:59:25,645 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:59:25,650 DEBUG HandlerThread:1971983 [system_info.py:__init__():26] System info init +2025-01-03 20:59:25,650 DEBUG HandlerThread:1971983 [system_info.py:__init__():41] System info init done +2025-01-03 20:59:25,650 INFO HandlerThread:1971983 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:59:25,650 INFO SystemMonitor:1971983 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:59:25,650 INFO HandlerThread:1971983 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:59:25,650 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:59:25,651 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:59:25,652 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:59:25,652 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:59:25,653 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:59:25,662 DEBUG HandlerThread:1971983 [system_info.py:probe():152] Probing system +2025-01-03 20:59:25,664 DEBUG HandlerThread:1971983 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:probe():200] Probing system done +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:59:25.662826', 'startedAt': '2025-01-04T01:59:25.347428', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3752.21675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.095, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 782.4653129577637}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:59:25,669 INFO HandlerThread:1971983 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:59:25,669 INFO HandlerThread:1971983 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:_save_code():44] Saving code +2025-01-03 20:59:25,670 DEBUG HandlerThread:1971983 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:59:25,670 DEBUG HandlerThread:1971983 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:59:25,732 DEBUG HandlerThread:1971983 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:59:25,733 INFO HandlerThread:1971983 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:59:25,736 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:59:25,969 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/motgmsf3-wandb-metadata.json +2025-01-03 20:59:25,981 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:59:25,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:59:25,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:25,984 DEBUG SenderThread:1971983 [sender.py:send():379] send: telemetry +2025-01-03 20:59:25,997 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/fl6n4962-code/FISH/eval_robot.py +2025-01-03 20:59:26,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:26,025 INFO HandlerThread:1971983 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/tb" + save: true +} +control { + relay_id: "e539364ef70f43b69840a643d82a51e8" +} +_info { + stream_id: "75aq4eke" +} + +2025-01-03 20:59:26,025 DEBUG HandlerThread:1971983 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:59:26,026 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:26,041 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/vwybxkh7-diff.patch +2025-01-03 20:59:26,165 DEBUG SenderThread:1971983 [sender.py:send():379] send: tbrecord +2025-01-03 20:59:26,165 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 20:59:26,165 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 with policy live +2025-01-03 20:59:26,383 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/rx0b9caa-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt +2025-01-03 20:59:26,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH +2025-01-03 20:59:26,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code +2025-01-03 20:59:26,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:27,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:28,584 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:28,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:29,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:30,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:30,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:31,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:32,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:33,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:34,649 DEBUG SenderThread:1971983 [sender.py:send():379] send: config +2025-01-03 20:59:34,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:35,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:35,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:36,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:36,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:37,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:38,587 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:38,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:39,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:40,587 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:40,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:40,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:41,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:41,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:41,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:42,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:43,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:44,589 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:44,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:45,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:46,589 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:46,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:46,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:47,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:48,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:49,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:50,590 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:50,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:51,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:51,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:52,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:53,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:54,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:55,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:55,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:56,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:56,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:57,050 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:57,592 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 20:59:57,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:58,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:59,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:00,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:01,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:02,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:02,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:03,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:04,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:05,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:06,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:07,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:08,595 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:08,701 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:08,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:09,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:10,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:10,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:11,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:11,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:12,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:13,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:14,077 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:14,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:15,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:16,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:17,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:18,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:19,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:19,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:20,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:21,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:22,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:23,976 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:00:23,976 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 with policy now +2025-01-03 21:00:23,978 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:00:23,979 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:00:23,979 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:00:23,980 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:00:23,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:24,258 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/i3h0y8w7-media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media +2025-01-03 21:00:24,896 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/23d6nyag-media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:24,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:25,653 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:00:25,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:00:25,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:25,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:26,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:26,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:26,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:27,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:28,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:28,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:29,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:30,455 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:30,601 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:30,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:31,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:32,602 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:32,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:33,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:34,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:35,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:35,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:36,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:37,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:38,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:39,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:40,604 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:00:40,877 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/zldt3mhn-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:00:40,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:40,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:40,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:41,054 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:41,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:42,605 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:42,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:43,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:44,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:45,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:46,606 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:46,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:46,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:47,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:48,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:49,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:50,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:51,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:51,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:52,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:53,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:54,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:55,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:00:55,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:55,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:56,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:56,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:57,123 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:57,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:58,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:59,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:00,611 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:00,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:01,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:02,612 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:02,813 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:02,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:03,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:04,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:05,999 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:07,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:07,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:08,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:09,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:10,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:10,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:10,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:11,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:12,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:13,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:13,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:14,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:15,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:16,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:17,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:18,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:18,794 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:19,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:20,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:20,284 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:01:20,284 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 with policy now +2025-01-03 21:01:20,291 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:01:20,291 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:01:20,291 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:01:20,292 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:01:20,545 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/q58v7jky-media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:20,617 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:01:20,617 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:20,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:20,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:01:20,892 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/m2yex6v5-media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:21,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:22,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:22,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:23,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:23,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:24,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:25,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:25,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:01:25,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:25,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:26,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:26,620 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:27,005 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:28,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:28,621 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:29,005 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:29,802 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:30,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:31,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:31,521 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:01:31,521 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:01:31,522 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:01:31,622 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:01:31,622 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:01:32,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:32,623 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:33,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:34,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:35,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:35,691 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:36,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:36,625 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:37,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:38,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:39,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:40,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:40,701 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:40,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:40,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:41,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:42,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:42,626 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:43,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:44,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:44,627 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:45,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:45,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:46,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:47,009 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:48,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:49,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:50,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:50,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:51,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:52,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:53,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:54,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:55,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:55,655 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:01:55,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:55,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:56,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:56,083 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:57,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:58,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:59,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:00,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:01,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:01,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:02,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:03,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:04,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:05,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:06,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:07,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:07,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:08,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:09,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:10,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:10,559 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:02:10,559 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1136994690265702739e.mp4 with policy now +2025-01-03 21:02:10,563 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:02:10,563 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:10,564 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:10,566 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:02:10,829 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/0ow3ds6h-media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:10,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:10,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:11,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:11,091 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/uyvzrxb8-media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:12,016 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:12,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:13,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:13,127 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:14,021 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:15,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:16,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:16,638 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:17,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:18,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:18,639 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:18,690 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:19,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:20,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:21,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:22,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:23,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:23,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:24,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:25,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:25,655 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:02:25,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:25,986 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:26,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:27,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:28,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:29,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:29,296 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:29,296 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:29,297 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:29,298 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:29,642 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:02:29,643 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:29,889 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/jklp17ag-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:02:30,027 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:30,643 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:31,027 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:32,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:33,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:34,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:34,644 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:34,737 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:35,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:36,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:37,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:38,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:39,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:39,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:40,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:40,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:40,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:41,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:42,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:43,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:44,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:45,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:45,064 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:46,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:47,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:48,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:49,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:50,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:50,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:51,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:52,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:53,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:54,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:55,030 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:55,454 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:02:55,454 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 with policy now +2025-01-03 21:02:55,457 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:02:55,457 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:55,457 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:55,458 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:55,651 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:55,651 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:55,652 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:02:55,656 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:02:55,750 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/kiil1o2x-media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:55,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:55,993 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:56,003 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/y207h7n6-media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:56,033 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:56,136 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:56,652 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:57,038 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:58,036 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:59,032 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:00,037 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:00,653 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:03:01,036 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:01,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:02,033 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:03,035 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:04,038 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:04,654 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:03:05,039 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:06,039 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:07,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:07,045 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:08,048 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:09,043 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:10,048 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:10,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:10,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:11,047 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:12,049 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:12,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:13,046 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:14,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:15,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:16,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:17,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:18,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:18,055 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:19,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:20,055 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:21,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:22,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:23,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:24,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:24,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:25,057 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:25,656 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:03:25,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:25,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:26,057 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:27,062 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:28,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:29,060 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:29,143 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:30,062 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:31,063 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:32,064 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:33,069 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:34,067 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:35,015 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:35,071 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:36,070 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:37,080 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:38,071 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:39,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:40,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:40,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:40,989 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:41,060 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:41,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:42,079 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:43,081 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:44,079 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:45,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:46,081 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:47,014 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:47,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:48,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:49,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:50,083 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:51,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:52,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:52,084 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:53,084 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:54,086 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:55,090 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:55,657 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:03:55,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:55,997 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:56,088 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:57,050 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:57,089 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:58,090 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:59,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:00,093 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:01,096 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:02,092 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:03,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:03,094 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:04,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:05,097 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:06,098 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:07,096 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:08,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:08,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:09,098 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:10,102 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:10,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:10,997 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:11,103 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:11,418 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:04:11,418 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:04:11,419 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:04:11,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:04:11,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:04:11,954 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/k9ly5vb7-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:04:12,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:12,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:13,107 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:13,829 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:14,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:15,106 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:16,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:16,678 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:17,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:18,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:18,679 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:19,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:19,523 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:20,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:21,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:22,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:23,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:24,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:25,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:25,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:25,657 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:04:25,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:25,990 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:26,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:27,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:28,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:29,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:30,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:30,129 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:31,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:32,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:33,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:34,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:35,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:36,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:36,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:37,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:38,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:39,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:40,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:40,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:40,990 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:41,086 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:41,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:42,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:43,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:44,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:44,688 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:45,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:46,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:47,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:47,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:48,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:49,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:50,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:51,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:52,119 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:52,446 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:52,493 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:04:52,493 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:04:52,494 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 with policy now +2025-01-03 21:04:52,494 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:04:52,494 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:04:52,494 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:52,692 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:04:52,759 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/ipkj68j3-media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:53,014 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/z8r1oppo-media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:53,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:54,123 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:54,692 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:55,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:55,658 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:04:56,003 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:56,003 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:56,130 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:57,141 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:58,133 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:58,265 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:58,693 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:59,135 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:00,149 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:00,694 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:01,150 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:02,144 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:03,151 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:03,575 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:03,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:04,148 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:04,459 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:05:04,460 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:05:04,680 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:05:04,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:04,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval +2025-01-03 21:05:04,908 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/otsy1mzd-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:04,936 INFO wandb-upload_2:1971983 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp9j1wpyh +2025-01-03 21:05:05,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:05:05,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:05:05,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:05,325 INFO SenderThread:1971983 [sender.py:send_request_log_artifact():1525] logged artifact run-75aq4eke-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg4NzI2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NjE2Mg==', 'latestArtifact': None}} +2025-01-03 21:05:05,325 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:05,325 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,326 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,326 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:05:05,326 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-03 21:05:05,326 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:05,327 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,330 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,330 DEBUG SenderThread:1971983 [sender.py:send():379] send: telemetry +2025-01-03 21:05:05,330 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,331 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,331 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,332 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,332 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,332 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,332 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,333 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,333 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,333 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,333 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,334 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,334 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,334 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,334 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,335 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,335 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:05:05,563 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/hubmczio-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:05,672 INFO wandb-upload_2:1971983 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjvfk30qd +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval +2025-01-03 21:05:06,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:06,185 INFO SenderThread:1971983 [sender.py:send_request_log_artifact():1525] logged artifact run-75aq4eke-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg4NzI4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NjE2NA==', 'latestArtifact': None}} +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:05:06,185 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json with policy now +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:06,186 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:06,357 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/eo5et8t7-media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:06,620 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/3l8by8bd-media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:06,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:06,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:07,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:08,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:09,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:09,186 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:10,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:10,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:05:10,993 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:05:11,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:12,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:13,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:14,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:15,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:15,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:15,666 DEBUG SenderThread:1971983 [sender.py:send():379] send: exit +2025-01-03 21:05:15,666 INFO SenderThread:1971983 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:05:15,666 INFO SenderThread:1971983 [sender.py:send_exit():588] handling runtime: 350 +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:send_exit():594] send defer +2025-01-03 21:05:15,667 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,667 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:05:15,667 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:05:15,668 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:05:15,668 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,668 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:05:15,668 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:05:15,668 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:05:15,668 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:05:15,668 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:05:15,669 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:05:15,669 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:05:15,700 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:05:15,721 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,721 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:05:15,722 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,722 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:05:16,700 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,507 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:05:22,507 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:22,508 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:22,508 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,508 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:05:22,509 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:22,509 DEBUG SenderThread:1971983 [sender.py:send():379] send: summary +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:22,513 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:05:22,513 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,513 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:05:22,513 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:05:22,517 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:22,633 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:05:22,633 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,633 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:05:22,634 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,634 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:05:22,667 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:22,703 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 21:05:22,703 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:26,383 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:05:26,383 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:26,384 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,384 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:05:26,384 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,384 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:05:26,384 INFO SenderThread:1971983 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:05:26,386 INFO SenderThread:1971983 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:05:26,389 INFO SenderThread:1971983 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:05:26,403 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:05:26,403 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,403 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:05:26,403 DEBUG SenderThread:1971983 [sender.py:send():379] send: artifact +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:05:26,609 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:05:26,669 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml config.yaml +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json wandb-summary.json +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt requirements.txt +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log output.log +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch diff.patch +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:05:26,708 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:26,708 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,708 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:05:26,710 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,710 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:05:26,710 INFO SenderThread:1971983 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:05:26,910 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 21:05:26,943 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt +2025-01-03 21:05:26,949 INFO wandb-upload_3:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/613xjrxn-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:26,957 INFO wandb-upload_4:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:26,965 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:27,166 INFO Thread-11 (_thread_body):1971983 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:05:27,166 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,166 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:05:27,166 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,166 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:05:27,166 INFO SenderThread:1971983 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:05:27,167 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:05:27,168 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,168 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:05:27,168 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,168 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:05:27,168 INFO SenderThread:1971983 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:05:27,360 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,360 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:05:27,360 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:05:27,361 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,361 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send():379] send: final +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send():379] send: footer +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,361 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:05:27,361 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:27,362 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:05:27,362 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:27,362 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:27,362 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:05:27,364 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:05:27,364 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:05:27,365 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:27,425 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:05:27,427 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:05:27,427 INFO HandlerThread:1971983 [handler.py:finish():882] shutting down handler +2025-01-03 21:05:28,364 INFO WriterThread:1971983 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [sender.py:finish():1615] shutting down sender +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [file_pusher.py:join():175] waiting for file pusher diff --git a/75aq4eke/wandb/debug.log b/75aq4eke/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9da8282812ddc6872928d3046297e39c7bc68040 --- /dev/null +++ b/75aq4eke/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Configure stats pid to 1971810 +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/settings +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug.log +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug-internal.log +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke'} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():619] starting backend +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():623] setting up manager +2025-01-03 20:59:25,355 INFO MainThread:1971810 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:59:25,356 INFO MainThread:1971810 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:59:25,369 INFO MainThread:1971810 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:59:25,376 INFO MainThread:1971810 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:59:25,598 INFO MainThread:1971810 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:59:25,640 INFO MainThread:1971810 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:59:25,640 INFO MainThread:1971810 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:59:25,982 INFO MainThread:1971810 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:59:25,985 INFO MainThread:1971810 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:59:25,985 INFO MainThread:1971810 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/tb, True +2025-01-03 20:59:34,648 INFO MainThread:1971810 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:05:28,427 WARNING MsgRouterThr:1971810 [router.py:message_loop():77] message_loop has been closed diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d08ed89e5b6e11a5e0332741d329ee261499fa4 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735955965 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..260fed1aee926a75f7c67b83fb8ec5b323d21693 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..703ec3d 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]} \ No newline at end of file diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7fea5f43677627ba583ed7e4e49d17eb6af6f2fe --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[13, 15, 0.8666666666666667, 0.5649803876746871, 0.9865136220078189, 0.6421056389524442, 0.982903074376562, 0.6818572878473872, 0.9611378013165401, 0.6947067603079482, 0.9614359735519694]]} \ No newline at end of file diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9253a06b2fecac86fc9e06dfb8d4c0b6c91dd2a0 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a512bbba28a2d223fb5f72c7ce2558a5d852f6f2ec72797975b5240755d770a7 +size 1450027 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..53edf35457f9050e0bb38d7b1b16d937a7aaec4f --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:320233e3aec19848df182bd08223f682721c3af9643e5ea1b17d6a45d1424e37 +size 1459964 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fcd377d17a1d7154570038ad46d67f770a22ac18 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1136994690265702739ef6163f5157efcef3fbe2b227a943e8f2cdf2c2a2d365 +size 1418203 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14f70accc2614c0b8a17163317e95cfd46245ca4 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036663fd181fb684915fdf998c256bd7355bd26df88fa0cfd2f7e4cc9fdfdaa9 +size 977247 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e403881bfc0ee66dfc602113166a5fd33c931a8 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a344ae7a6fd7d295455849ef0845435731ab099787293c1f6421d6969325b54 +size 1546270 diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1a7ac373128390b95182760aa396d7c2c67a703e --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log @@ -0,0 +1,278 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735955974.649252]: resetting environment +[INFO] [1735955974.655859]: cleared current plan +[INFO] [1735955974.656771]: moving to home +[INFO] [1735955975.659421]: reached home +[INFO] [1735955975.660806]: reset action history +[INFO] [1735955978.371097]: environment reset +Starting episode 0 +[INFO] [1735955978.375414]: resetting environment +[INFO] [1735955978.384202]: cleared current plan +[INFO] [1735955978.385333]: moving to home +[INFO] [1735955979.388797]: reached home +[INFO] [1735955979.390655]: reset action history +[INFO] [1735955982.099612]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735955984.488767]: resetting environment +[INFO] [1735955984.492746]: cleared current plan +[INFO] [1735955984.493588]: moving to home +[INFO] [1735955985.496709]: reached home +[INFO] [1735955985.499605]: reset action history +[INFO] [1735955988.206754]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/episode_rosbags/episode_0_2025-01-03-20-59-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1971810.log +started roslaunch server http://158.130.50.37:45587/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1972089] +process[print_text-2]: started with pid [1972090] +process[rosbag_record-3]: started with pid [1972114] +started bagging! +For topic gripper_width: timestamp difference is -114818725 for nearest: 1735956006461585608 - target: 1735956006576404333 at idx 237 +For topic gripper_width: timestamp difference is 108922173 for nearest: 1735956022228288580 - target: 1735956022119366407 at idx 249 +[INFO] [1735956022.991037]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735956023.760969]: Stored episode 1. +[INFO] [1735956023.761679]: Saving video... +[INFO] [1735956023.959285]: Video saved! +Starting episode 1 +[INFO] [1735956023.979259]: resetting environment +[INFO] [1735956023.986627]: cleared current plan +[INFO] [1735956023.987041]: moving to home +[INFO] [1735956027.689377]: reached home +[INFO] [1735956027.689860]: reset action history +[INFO] [1735956030.451619]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956040.156065]: resetting environment +[INFO] [1735956040.159331]: cleared current plan +[INFO] [1735956040.159489]: moving to home +[INFO] [1735956041.161344]: reached home +[INFO] [1735956041.162754]: reset action history +[INFO] [1735956043.872059]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/episode_rosbags/episode_1_2025-01-03-21-00-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1971810.log +started roslaunch server http://158.130.50.37:32827/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1972239] +process[print_text-5]: started with pid [1972240] +process[rosbag_record-6]: started with pid [1972241] +started bagging! +For topic gripper_width: timestamp difference is 106562253 for nearest: 1735956059428255197 - target: 1735956059321692944 at idx 198 +For topic gripper_width: timestamp difference is 111331274 for nearest: 1735956061761589577 - target: 1735956061650258303 at idx 225 +[INFO] [1735956078.794442]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735956079.514589]: Stored episode 2. +[INFO] [1735956079.514805]: Saving video... +[WARN] [1735956080.140870]: Plan exhausted +[WARN] [1735956080.176116]: Plan exhausted +[WARN] [1735956080.216071]: Plan exhausted +[WARN] [1735956080.256277]: Plan exhausted +[WARN] [1735956080.296186]: Plan exhausted +[INFO] [1735956080.250504]: Video saved! +Starting episode 2 +[INFO] [1735956080.293734]: resetting environment +[INFO] [1735956080.306299]: cleared current plan +[INFO] [1735956080.306571]: moving to home +[INFO] [1735956084.011645]: reached home +[INFO] [1735956084.019573]: reset action history +[INFO] [1735956086.754096]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956090.661680]: resetting environment +[INFO] [1735956090.675943]: cleared current plan +[INFO] [1735956090.676164]: moving to home +[INFO] [1735956091.677228]: reached home +[INFO] [1735956091.677636]: reset action history +[INFO] [1735956094.385150]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/episode_rosbags/episode_2_2025-01-03-21-01-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1971810.log +started roslaunch server http://158.130.50.37:41141/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1972383] +process[print_text-8]: started with pid [1972384] +process[rosbag_record-9]: started with pid [1972408] +started bagging! +For topic gripper_width: timestamp difference is 105679988 for nearest: 1735956100628368854 - target: 1735956100522688866 at idx 72 +For topic gripper_width: timestamp difference is 106252918 for nearest: 1735956101428303251 - target: 1735956101322050333 at idx 85 +For topic gripper_width: timestamp difference is 107905332 for nearest: 1735956103528256122 - target: 1735956103420350790 at idx 116 +[INFO] [1735956129.373248]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735956130.121776]: Stored episode 3. +[INFO] [1735956130.125553]: Saving video... +[INFO] [1735956130.532814]: Video saved! +Starting episode 3 +[INFO] [1735956130.572656]: resetting environment +[INFO] [1735956130.595783]: cleared current plan +[INFO] [1735956130.596042]: moving to home +[INFO] [1735956133.901266]: reached home +[INFO] [1735956133.905886]: reset action history +[INFO] [1735956136.639855]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956148.734985]: resetting environment +[INFO] [1735956148.746996]: cleared current plan +[INFO] [1735956148.747288]: moving to home +[INFO] [1735956149.749134]: reached home +[INFO] [1735956149.750289]: reset action history +[INFO] [1735956152.473553]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/episode_rosbags/episode_3_2025-01-03-21-02-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1971810.log +started roslaunch server http://158.130.50.37:42957/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1972567] +process[print_text-11]: started with pid [1972568] +process[rosbag_record-12]: started with pid [1972569] +started bagging! +[WARN] [1735956175.253787]: Plan exhausted +[WARN] [1735956175.290973]: Plan exhausted +[WARN] [1735956175.330396]: Plan exhausted +[WARN] [1735956175.374039]: Plan exhausted +[WARN] [1735956175.412356]: Plan exhausted +[WARN] [1735956175.450739]: Plan exhausted +preempting episode +[INFO] [1735956174.082412]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735956174.917478]: Stored episode 4. +[INFO] [1735956174.918961]: Saving video... +[INFO] [1735956175.422570]: Video saved! +Starting episode 4 +[INFO] [1735956175.461600]: resetting environment +[INFO] [1735956175.487628]: cleared current plan +[INFO] [1735956175.488196]: moving to home +[INFO] [1735956179.191350]: reached home +[INFO] [1735956179.192192]: reset action history +[INFO] [1735956181.986637]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956250.727295]: resetting environment +[INFO] [1735956250.773810]: cleared current plan +[INFO] [1735956250.777838]: moving to home +[INFO] [1735956251.784183]: reached home +[INFO] [1735956251.793189]: reset action history +[INFO] [1735956254.521345]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/episode_rosbags/episode_4_2025-01-03-21-04-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1971810.log +started roslaunch server http://158.130.50.37:37653/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1972800] +process[print_text-14]: started with pid [1972801] +process[rosbag_record-15]: started with pid [1972802] +started bagging! +For topic gripper_width: timestamp difference is 121750761 for nearest: 1735956283628253628 - target: 1735956283506502867 at idx 249 +[INFO] [1735956291.158666]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735956291.937767]: Stored episode 5. +[INFO] [1735956291.948669]: Saving video... +[WARN] [1735956292.493283]: Plan exhausted +[INFO] [1735956292.442499]: Video saved! +[INFO] [1735956292.489013]: resetting environment +[INFO] [1735956292.513477]: cleared current plan +[INFO] [1735956292.513788]: moving to home +[INFO] [1735956296.221788]: reached home +[INFO] [1735956296.231744]: reset action history +[INFO] [1735956299.021725]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..af38e8850457fe7575a5360e1caeac759b7dc124 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:59:25.662826", + "startedAt": "2025-01-04T01:59:25.347428", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3752.21675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4502.095, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.021, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 782.4653129577637 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3f148ecfbe8dfdb3d8fae6b28eb7cfbd35f6f520 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a512bbba28a2d223fb5f72c7ce2558a5d852f6f2ec72797975b5240755d770a7", "size": 1450027, "path": "media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4"}, "global_step": 613, "_timestamp": 1735956304.9350116, "_runtime": 339.57858657836914, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "320233e3aec19848df182bd08223f682721c3af9643e5ea1b17d6a45d1424e37", "size": 1459964, "path": "media/videos/eval/1_eval_1_320233e3aec19848df18.mp4"}, "eval/num_success": 3.0, "episode": 15.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "1136994690265702739ef6163f5157efcef3fbe2b227a943e8f2cdf2c2a2d365", "size": 1418203, "path": "media/videos/eval/2_eval_2_1136994690265702739e.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "036663fd181fb684915fdf998c256bd7355bd26df88fa0cfd2f7e4cc9fdfdaa9", "size": 977247, "path": "media/videos/eval/3_eval_3_036663fd181fb684915f.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8a344ae7a6fd7d295455849ef0845435731ab099787293c1f6421d6969325b54", "size": 1546270, "path": "media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://6khb38pquom5wmfl5uc47gbw9p3qbiwkznyrrxz0oxwt9y9fp2go9rwq0dtmd1ohd1zzbr15gvbip9tr32x1arg05hehqzc8orfh195ewzu0dckomwjp5capsowk2e4t/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://q7y1rusl9828k8n7lleur0b4oz1vfww8wg4r7bya230w10yaoy62hx1l3u7ph858ewbvg1iu1iow3dc8dexegpe41cqu2v64hbjhpotp2wmtopw5i4yj9kdkass64zva:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 13, "total_num_episodes": 15, "total_success_rate": 0.8666666666666667, "total_success_rate_lb_0.01": 0.5649803876746871, "total_success_rate_ub_0.01": 0.9865136220078189, "total_success_rate_lb_0.025": 0.6421056389524442, "total_success_rate_ub_0.025": 0.982903074376562, "total_success_rate_lb_0.05": 0.6818572878473872, "total_success_rate_ub_0.05": 0.9611378013165401, "total_success_rate_lb_0.1": 0.6947067603079482, "total_success_rate_ub_0.1": 0.9614359735519694, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "6e34c8961e235c47938ca7eccc01706651cc4e51f3175c940c14b8aa686cfbb0", "size": 519, "artifact_path": "wandb-client-artifact://nsxx8bwxrcnj5mziscmdma6rqr8nv1zy90bbqihultyzgz7tv2k7u44a5ugipe8r9mhi9jkvgz5uvxi5231ewqzf3bzgfiysjj18pp87ok1wnnzpe2mf5j7ex9qple0t/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://m4l79bja8hdfywmlo8rvvv2s6c5ta17w5qzm14unu6gd46hykwlpilcdvwa6gm3a7rpnnx5p6eirdsep74g85o4hoaqvj1kqamqpz4x49h4tslwuibcb2xfwhkbwu9w0:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 350}, "eval/total_success_rate": 0.8666666746139526, "eval/total_success_rate_lb_0.01": 0.5649803876876831, "eval/total_success_rate_ub_0.01": 0.986513614654541, "eval/total_success_rate_lb_0.025": 0.6421056389808655, "eval/total_success_rate_ub_0.025": 0.9829030632972717, "eval/total_success_rate_lb_0.05": 0.6818572878837585, "eval/total_success_rate_ub_0.05": 0.9611377716064453, "eval/total_success_rate_lb_0.1": 0.6947067379951477, "eval/total_success_rate_ub_0.1": 0.9614359736442566} \ No newline at end of file diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug-internal.log b/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f5dad30a043a8d044fe8751ef0993621433c61ba --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug-internal.log @@ -0,0 +1,879 @@ +2025-01-03 20:59:25,356 INFO StreamThr :1971983 [internal.py:wandb_internal():85] W&B internal server running at pid: 1971983, started at: 2025-01-03 20:59:25.355968 +2025-01-03 20:59:25,357 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:59:25,370 INFO WriterThread:1971983 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb +2025-01-03 20:59:25,370 DEBUG SenderThread:1971983 [sender.py:send():379] send: header +2025-01-03 20:59:25,378 DEBUG SenderThread:1971983 [sender.py:send():379] send: run +2025-01-03 20:59:25,583 INFO SenderThread:1971983 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files +2025-01-03 20:59:25,583 INFO SenderThread:1971983 [sender.py:_start_run_threads():1188] run started: 75aq4eke with start time 1735955965.356425 +2025-01-03 20:59:25,599 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:59:25,599 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:59:25,645 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:59:25,650 DEBUG HandlerThread:1971983 [system_info.py:__init__():26] System info init +2025-01-03 20:59:25,650 DEBUG HandlerThread:1971983 [system_info.py:__init__():41] System info init done +2025-01-03 20:59:25,650 INFO HandlerThread:1971983 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:59:25,650 INFO SystemMonitor:1971983 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:59:25,650 INFO HandlerThread:1971983 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:59:25,650 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:59:25,651 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:59:25,652 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:59:25,652 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:59:25,653 INFO SystemMonitor:1971983 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:59:25,662 DEBUG HandlerThread:1971983 [system_info.py:probe():152] Probing system +2025-01-03 20:59:25,664 DEBUG HandlerThread:1971983 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:probe():200] Probing system done +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:59:25.662826', 'startedAt': '2025-01-04T01:59:25.347428', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3752.21675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.095, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 782.4653129577637}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:59:25,669 INFO HandlerThread:1971983 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:59:25,669 INFO HandlerThread:1971983 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:59:25,669 DEBUG HandlerThread:1971983 [system_info.py:_save_code():44] Saving code +2025-01-03 20:59:25,670 DEBUG HandlerThread:1971983 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:59:25,670 DEBUG HandlerThread:1971983 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:59:25,732 DEBUG HandlerThread:1971983 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:59:25,733 INFO HandlerThread:1971983 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:59:25,736 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:59:25,736 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:59:25,969 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/motgmsf3-wandb-metadata.json +2025-01-03 20:59:25,981 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:59:25,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:59:25,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:25,984 DEBUG SenderThread:1971983 [sender.py:send():379] send: telemetry +2025-01-03 20:59:25,997 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/fl6n4962-code/FISH/eval_robot.py +2025-01-03 20:59:26,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:26,025 INFO HandlerThread:1971983 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/tb" + save: true +} +control { + relay_id: "e539364ef70f43b69840a643d82a51e8" +} +_info { + stream_id: "75aq4eke" +} + +2025-01-03 20:59:26,025 DEBUG HandlerThread:1971983 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:59:26,026 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:26,041 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/vwybxkh7-diff.patch +2025-01-03 20:59:26,165 DEBUG SenderThread:1971983 [sender.py:send():379] send: tbrecord +2025-01-03 20:59:26,165 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 20:59:26,165 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 with policy live +2025-01-03 20:59:26,383 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/rx0b9caa-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json +2025-01-03 20:59:26,585 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt +2025-01-03 20:59:26,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH +2025-01-03 20:59:26,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code +2025-01-03 20:59:26,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:27,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:28,584 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:28,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:29,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:30,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:30,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:31,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:32,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:33,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:34,649 DEBUG SenderThread:1971983 [sender.py:send():379] send: config +2025-01-03 20:59:34,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:35,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:35,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:36,586 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:36,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:37,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:38,587 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:38,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:39,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:40,587 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:40,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:40,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:41,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:41,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:41,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:42,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:43,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:44,589 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:44,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:45,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:46,589 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:46,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:46,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:47,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:48,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:49,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:50,590 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 20:59:50,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:51,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:51,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:52,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:53,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:54,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:55,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:59:55,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:59:56,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:56,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:57,050 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:59:57,592 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 20:59:57,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:58,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:59:59,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:00,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:01,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:02,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:02,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:03,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:04,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:05,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:06,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:07,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:08,595 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:08,701 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:08,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:09,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:10,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:10,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:11,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:11,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:12,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:13,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:14,077 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:14,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:15,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:16,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:17,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:18,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:19,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:19,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:20,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:21,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:22,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:23,976 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:00:23,976 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 with policy now +2025-01-03 21:00:23,978 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:00:23,979 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:00:23,979 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:00:23,980 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:00:23,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:24,258 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/i3h0y8w7-media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,599 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos +2025-01-03 21:00:24,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media +2025-01-03 21:00:24,896 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/23d6nyag-media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:00:24,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:24,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:25,653 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:00:25,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:00:25,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:25,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:26,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:26,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:26,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:27,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:28,600 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:28,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:29,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:30,455 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:30,601 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:30,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:31,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:32,602 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:32,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:33,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:34,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:35,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:35,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:36,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:37,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:38,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:39,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:40,604 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:00:40,877 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/zldt3mhn-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:00:40,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:40,982 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:40,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:41,054 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:41,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:42,605 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:42,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:43,995 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:44,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:45,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:46,606 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:00:46,984 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:46,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:47,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:48,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:49,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:50,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:51,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:51,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:52,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:53,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:54,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:55,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:00:55,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:00:55,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:00:56,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:56,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:57,123 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:00:57,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:58,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:00:59,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:00,611 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:00,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:01,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:02,612 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:02,813 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:02,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:03,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:04,998 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:05,999 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:07,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:07,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:08,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:09,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:10,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:10,982 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:10,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:11,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:12,000 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:13,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:13,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:14,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:15,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:16,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:17,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:18,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:18,794 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:19,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:20,001 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:20,284 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:01:20,284 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 with policy now +2025-01-03 21:01:20,291 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:01:20,291 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:01:20,291 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:01:20,292 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:01:20,545 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/q58v7jky-media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:20,617 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:01:20,617 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:20,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:20,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:01:20,892 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/m2yex6v5-media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:01:21,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:22,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:22,618 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:23,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:23,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:24,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:25,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:25,654 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:01:25,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:25,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:26,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:26,620 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:27,005 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:28,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:28,621 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:29,005 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:29,802 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:30,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:31,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:31,521 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:01:31,521 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:01:31,522 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:01:31,622 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:01:31,622 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:01:32,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:32,623 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:33,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:34,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:35,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:35,691 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:36,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:36,625 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:37,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:38,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:39,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:40,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:40,701 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:40,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:40,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:41,007 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:42,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:42,626 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:43,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:44,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:44,627 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:01:45,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:45,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:46,008 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:47,009 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:48,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:49,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:50,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:50,989 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:51,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:52,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:53,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:54,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:55,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:55,655 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:01:55,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:01:55,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:01:56,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:56,083 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:01:57,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:58,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:01:59,011 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:00,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:01,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:01,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:02,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:03,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:04,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:05,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:06,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:07,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:07,991 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:08,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:09,012 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:10,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:10,559 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:02:10,559 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1136994690265702739e.mp4 with policy now +2025-01-03 21:02:10,563 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:02:10,563 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:10,564 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:10,566 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:10,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:02:10,829 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/0ow3ds6h-media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:10,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:10,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:11,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:11,091 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/uyvzrxb8-media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:02:12,016 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:12,636 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:13,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:13,127 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:14,021 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:15,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:16,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:16,638 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:17,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:18,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:18,639 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:18,690 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:19,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:20,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:21,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:22,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:23,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:23,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:24,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:25,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:25,655 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:02:25,985 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:25,986 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:26,029 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:27,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:28,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:29,022 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:29,296 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:29,296 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:29,297 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:29,298 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:29,642 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:02:29,643 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:29,889 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/jklp17ag-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:02:30,027 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:30,643 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:31,027 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:32,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:33,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:34,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:34,644 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:34,737 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:35,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:36,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:37,024 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:38,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:39,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:39,994 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:40,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:40,983 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:40,983 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:41,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:42,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:43,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:44,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:45,025 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:45,064 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:46,026 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:47,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:48,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:49,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:50,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:50,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:51,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:52,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:53,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:54,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:55,030 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:55,454 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:02:55,454 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 with policy now +2025-01-03 21:02:55,457 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:02:55,457 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:02:55,457 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:02:55,458 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:02:55,651 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:02:55,651 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:55,652 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:02:55,656 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:02:55,750 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/kiil1o2x-media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:55,993 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:02:55,993 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:02:56,003 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/y207h7n6-media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:02:56,033 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:56,136 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:02:56,652 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:02:57,038 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:58,036 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:02:59,032 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:00,037 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:00,653 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:03:01,036 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:01,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:02,033 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:03,035 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:04,038 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:04,654 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:03:05,039 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:06,039 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:07,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:07,045 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:08,048 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:09,043 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:10,048 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:10,986 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:10,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:11,047 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:12,049 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:12,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:13,046 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:14,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:15,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:16,052 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:17,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:18,010 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:18,055 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:19,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:20,055 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:21,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:22,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:23,056 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:24,006 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:24,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:25,057 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:25,656 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:03:25,987 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:25,987 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:26,057 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:27,062 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:28,058 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:29,060 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:29,143 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:30,062 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:31,063 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:32,064 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:33,069 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:34,067 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:35,015 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:35,071 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:36,070 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:37,080 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:38,071 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:39,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:40,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:40,988 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:40,989 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:41,060 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:41,073 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:42,079 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:43,081 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:44,079 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:45,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:46,081 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:47,014 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:47,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:48,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:49,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:50,083 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:51,085 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:52,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:52,084 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:53,084 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:54,086 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:55,090 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:55,657 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:03:55,996 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:03:55,997 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:03:56,088 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:57,050 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:03:57,089 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:58,090 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:03:59,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:00,093 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:01,096 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:02,092 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:03,017 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:03,094 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:04,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:05,097 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:06,098 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:07,096 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:08,020 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:08,095 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:09,098 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:10,102 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:10,997 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:10,997 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:11,103 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:11,418 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:04:11,418 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:04:11,419 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:04:11,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:04:11,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:04:11,954 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/k9ly5vb7-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:04:12,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:12,676 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:13,107 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:13,829 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:14,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:15,106 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:16,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:16,678 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:17,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:18,109 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:18,679 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:19,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:19,523 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:20,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:21,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:22,110 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:23,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:24,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:25,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:25,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:25,657 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:04:25,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:25,990 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:26,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:27,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:28,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:29,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:30,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:30,129 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:31,111 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:32,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:33,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:34,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:35,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:36,018 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:36,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:37,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:38,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:39,112 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:40,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:40,990 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:40,990 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:41,086 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:41,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:42,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:43,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:44,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:44,688 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:45,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:46,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:47,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:47,113 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:48,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:49,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:50,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:51,114 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:52,119 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:52,446 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:52,493 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:04:52,493 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:04:52,494 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 with policy now +2025-01-03 21:04:52,494 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:04:52,494 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:04:52,494 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:52,691 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:52,692 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval +2025-01-03 21:04:52,759 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/ipkj68j3-media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:53,014 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/z8r1oppo-media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:04:53,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:54,123 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:54,692 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:55,124 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:55,658 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:04:56,003 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:04:56,003 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:04:56,130 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:57,141 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:58,133 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:04:58,265 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:04:58,693 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:04:59,135 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:00,149 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:00,694 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:01,150 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:02,144 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:03,151 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:03,575 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:03,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:04,148 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:04,459 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:05:04,460 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:05:04,680 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:05:04,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:04,695 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table +2025-01-03 21:05:04,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval +2025-01-03 21:05:04,908 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/otsy1mzd-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:04,936 INFO wandb-upload_2:1971983 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpp9j1wpyh +2025-01-03 21:05:05,019 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:05:05,028 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:05:05,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:05,325 INFO SenderThread:1971983 [sender.py:send_request_log_artifact():1525] logged artifact run-75aq4eke-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg4NzI2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NjE2Mg==', 'latestArtifact': None}} +2025-01-03 21:05:05,325 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:05,325 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,326 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,326 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:05:05,326 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-03 21:05:05,326 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:05,327 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,330 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,330 DEBUG SenderThread:1971983 [sender.py:send():379] send: telemetry +2025-01-03 21:05:05,330 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,331 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,331 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,331 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,332 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,332 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,332 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,332 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,333 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,333 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,333 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,333 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,334 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,334 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,334 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,334 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,335 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:05,335 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:05,335 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:05:05,563 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/hubmczio-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:05,672 INFO wandb-upload_2:1971983 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjvfk30qd +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:05,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval +2025-01-03 21:05:06,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:06,185 INFO SenderThread:1971983 [sender.py:send_request_log_artifact():1525] logged artifact run-75aq4eke-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg4NzI4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NjE2NA==', 'latestArtifact': None}} +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send():379] send: files +2025-01-03 21:05:06,185 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json with policy now +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:06,185 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:06,186 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:06,357 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/eo5et8t7-media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:06,620 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/3l8by8bd-media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:06,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:06,696 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:07,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:08,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:09,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:09,186 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:10,145 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:10,992 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:05:10,993 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:05:11,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:12,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:13,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:14,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:15,082 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:15,146 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:15,666 DEBUG SenderThread:1971983 [sender.py:send():379] send: exit +2025-01-03 21:05:15,666 INFO SenderThread:1971983 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:05:15,666 INFO SenderThread:1971983 [sender.py:send_exit():588] handling runtime: 350 +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:send_exit():594] send defer +2025-01-03 21:05:15,667 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,667 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:05:15,667 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:05:15,667 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:05:15,668 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:05:15,668 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,668 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:05:15,668 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:05:15,668 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:05:15,668 INFO HandlerThread:1971983 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:05:15,668 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:05:15,668 DEBUG SystemMonitor:1971983 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:05:15,669 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:05:15,669 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:05:15,700 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:05:15,720 INFO HandlerThread:1971983 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send():379] send: stats +2025-01-03 21:05:15,721 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,721 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:05:15,721 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:05:15,721 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:05:15,722 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:15,722 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:05:16,700 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:22,506 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,507 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:05:22,507 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:22,507 DEBUG SenderThread:1971983 [sender.py:send():379] send: history +2025-01-03 21:05:22,508 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:05:22,508 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,508 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:05:22,509 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:22,509 DEBUG SenderThread:1971983 [sender.py:send():379] send: summary +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:05:22,513 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:05:22,513 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,513 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:05:22,513 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,513 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:05:22,517 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:05:22,633 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:05:22,633 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:22,633 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:05:22,634 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:22,634 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:05:22,667 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:22,703 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 21:05:22,703 INFO Thread-12 :1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:26,383 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:05:26,383 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:26,384 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,384 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:05:26,384 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,384 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:05:26,384 INFO SenderThread:1971983 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:05:26,386 INFO SenderThread:1971983 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:05:26,389 INFO SenderThread:1971983 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:05:26,403 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:05:26,403 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,403 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:05:26,403 DEBUG SenderThread:1971983 [sender.py:send():379] send: artifact +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:05:26,609 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:05:26,609 INFO SenderThread:1971983 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:05:26,669 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml config.yaml +2025-01-03 21:05:26,704 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json wandb-summary.json +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt requirements.txt +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:05:26,706 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log output.log +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/diff.patch diff.patch +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 media/videos/eval/3_eval_3_036663fd181fb684915f.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 media/videos/eval/4_eval_4_8a344ae7a6fd7d295455.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/2_eval_2_1136994690265702739e.mp4 media/videos/eval/2_eval_2_1136994690265702739e.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 media/videos/eval/0_eval_0_a512bbba28a2d223fb5f.mp4 +2025-01-03 21:05:26,707 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 media/videos/eval/1_eval_1_320233e3aec19848df18.mp4 +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json media/table/eval/total_success_rate_ci_6_6e34c8961e235c47938c.table.json +2025-01-03 21:05:26,708 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:05:26,708 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:26,708 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:26,708 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:05:26,710 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:26,710 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:05:26,710 INFO SenderThread:1971983 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:05:26,910 INFO wandb-upload_0:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/config.yaml +2025-01-03 21:05:26,943 INFO wandb-upload_2:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/requirements.txt +2025-01-03 21:05:26,949 INFO wandb-upload_3:1971983 [upload_job.py:push():130] Uploaded file /tmp/tmpa3_yy03vwandb/613xjrxn-events.out.tfevents.1735955965.leonmkim-ROG-Strix-G15CS-G15CS.1971810.0 +2025-01-03 21:05:26,957 INFO wandb-upload_4:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/wandb-summary.json +2025-01-03 21:05:26,965 INFO wandb-upload_1:1971983 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/files/output.log +2025-01-03 21:05:27,166 INFO Thread-11 (_thread_body):1971983 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:05:27,166 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,166 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:05:27,166 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,166 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:05:27,166 INFO SenderThread:1971983 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:05:27,167 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:05:27,168 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,168 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:05:27,168 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,168 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:05:27,168 INFO SenderThread:1971983 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:05:27,360 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,360 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:05:27,360 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:05:27,360 INFO SenderThread:1971983 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:05:27,361 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:05:27,361 INFO HandlerThread:1971983 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send():379] send: final +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send():379] send: footer +2025-01-03 21:05:27,361 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: defer +2025-01-03 21:05:27,361 INFO SenderThread:1971983 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:05:27,361 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:27,362 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:05:27,362 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:05:27,362 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:27,362 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:05:27,364 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:05:27,364 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:05:27,365 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:05:27,425 DEBUG SenderThread:1971983 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:05:27,426 INFO MainThread:1971983 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:05:27,427 DEBUG HandlerThread:1971983 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:05:27,427 INFO HandlerThread:1971983 [handler.py:finish():882] shutting down handler +2025-01-03 21:05:28,364 INFO WriterThread:1971983 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [sender.py:finish():1615] shutting down sender +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:05:28,426 INFO SenderThread:1971983 [file_pusher.py:join():175] waiting for file pusher diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug.log b/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9da8282812ddc6872928d3046297e39c7bc68040 --- /dev/null +++ b/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Configure stats pid to 1971810 +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/settings +2025-01-03 20:59:25,350 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug.log +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/wandb/run-20250103_205925-75aq4eke/logs/debug-internal.log +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke'} +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():619] starting backend +2025-01-03 20:59:25,351 INFO MainThread:1971810 [wandb_init.py:init():623] setting up manager +2025-01-03 20:59:25,355 INFO MainThread:1971810 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:59:25,356 INFO MainThread:1971810 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:59:25,369 INFO MainThread:1971810 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:59:25,376 INFO MainThread:1971810 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:59:25,598 INFO MainThread:1971810 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:59:25,640 INFO MainThread:1971810 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:59:25,640 INFO MainThread:1971810 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:59:25,982 INFO MainThread:1971810 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:59:25,983 INFO MainThread:1971810 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:59:25,985 INFO MainThread:1971810 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:59:25,985 INFO MainThread:1971810 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/75aq4eke/tb, True +2025-01-03 20:59:34,648 INFO MainThread:1971810 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:05:28,427 WARNING MsgRouterThr:1971810 [router.py:message_loop():77] message_loop has been closed diff --git a/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb b/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb new file mode 100644 index 0000000000000000000000000000000000000000..17f6b25298f34bd5ee22c357d702592244069873 Binary files /dev/null and b/75aq4eke/wandb/run-20250103_205925-75aq4eke/run-75aq4eke.wandb differ diff --git a/97ao8n2y/.hydra/config.yaml b/97ao8n2y/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/97ao8n2y/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/97ao8n2y/.hydra/hydra.yaml b/97ao8n2y/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..798e35829fab1fcf634938ab2d75c2fbdfc887f1 --- /dev/null +++ b/97ao8n2y/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/97ao8n2y/.hydra/overrides.yaml b/97ao8n2y/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/97ao8n2y/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/97ao8n2y/episode_rosbags/aligned_depth_to_color_K.npy b/97ao8n2y/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/97ao8n2y/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/97ao8n2y/episode_rosbags/color_K.npy b/97ao8n2y/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/97ao8n2y/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/97ao8n2y/episode_rosbags/episode_1_2025-01-04-21-44-44.bag b/97ao8n2y/episode_rosbags/episode_1_2025-01-04-21-44-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..2fc10facec120c2535bf4dde4d83e38cf427ac19 --- /dev/null +++ b/97ao8n2y/episode_rosbags/episode_1_2025-01-04-21-44-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5168781496fc7c00ac86b3e0292e654a03002deba51eafa5c8464476f61b989a +size 2609763383 diff --git a/97ao8n2y/episode_rosbags/episode_4_2025-01-04-21-47-32.bag b/97ao8n2y/episode_rosbags/episode_4_2025-01-04-21-47-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..81cc1a1f18d0ef88f0a0eb9bbd288d17d47fe6fa --- /dev/null +++ b/97ao8n2y/episode_rosbags/episode_4_2025-01-04-21-47-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f3d0a4e9c622118a24eff34de271d9c69bf69f52a5da525176bf1314b4abff +size 2608882078 diff --git a/97ao8n2y/eval_video/0_eval.mp4 b/97ao8n2y/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..234b21dc8bcc582b9c58fc101e014c4925eb06ef --- /dev/null +++ b/97ao8n2y/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776ba6f8d1335d98898ca0dfb30b082dfb658f37cb805cc42a43084f746cad53 +size 1449040 diff --git a/97ao8n2y/eval_video/1_eval.mp4 b/97ao8n2y/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ee88c73c9192765bb0121660ac5170df5258ced --- /dev/null +++ b/97ao8n2y/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6562e4bf5aa0b73485524dc80dce5b9e0cb31adde15a92cfb28904023cd41e32 +size 1464995 diff --git a/97ao8n2y/eval_video/2_eval.mp4 b/97ao8n2y/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9e501c7d8ce6afff99a7cd0011e627bdf3e0ba3 --- /dev/null +++ b/97ao8n2y/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb26fcea183c3ae0aa38a4c46f5db42fa15333496efe601e273b9f1995f5e563 +size 1459054 diff --git a/97ao8n2y/eval_video/3_eval.mp4 b/97ao8n2y/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5be2aab9b9da2266f3fc5dd2c2e7dfcf1668350 --- /dev/null +++ b/97ao8n2y/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4651ece086df075e5442a24ab765a98e915558ed88e888a3c7088f53c4e12966 +size 1467788 diff --git a/97ao8n2y/eval_video/4_eval.mp4 b/97ao8n2y/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f76203a0dba3d04c48c79f6fc505f2c835e80c0 --- /dev/null +++ b/97ao8n2y/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d451a7b8e4e09dc6abf67cb046aeee60f2e9a9dc12acf4e62f2182a9395df0f9 +size 1466610 diff --git a/97ao8n2y/tb/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 b/97ao8n2y/tb/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 new file mode 100644 index 0000000000000000000000000000000000000000..6adce958ced6eb0cf6af346da43347a320761c79 --- /dev/null +++ b/97ao8n2y/tb/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d26c4f0163a7a960b8f8bf5fbc15eab73210317d96e373dddd8ef3e20b8aab +size 6539 diff --git a/97ao8n2y/wandb/debug-internal.log b/97ao8n2y/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1c138e7395ee3f58db92b05f381c01a7f5fbb86d --- /dev/null +++ b/97ao8n2y/wandb/debug-internal.log @@ -0,0 +1,833 @@ +2025-01-04 21:43:28,261 INFO StreamThr :2030562 [internal.py:wandb_internal():85] W&B internal server running at pid: 2030562, started at: 2025-01-04 21:43:28.260854 +2025-01-04 21:43:28,262 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:43:28,274 INFO WriterThread:2030562 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb +2025-01-04 21:43:28,274 DEBUG SenderThread:2030562 [sender.py:send():379] send: header +2025-01-04 21:43:28,286 DEBUG SenderThread:2030562 [sender.py:send():379] send: run +2025-01-04 21:43:28,532 INFO SenderThread:2030562 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files +2025-01-04 21:43:28,532 INFO SenderThread:2030562 [sender.py:_start_run_threads():1188] run started: 97ao8n2y with start time 1736045008.263183 +2025-01-04 21:43:28,547 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:43:28,547 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:43:28,621 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:43:28,626 DEBUG HandlerThread:2030562 [system_info.py:__init__():26] System info init +2025-01-04 21:43:28,626 DEBUG HandlerThread:2030562 [system_info.py:__init__():41] System info init done +2025-01-04 21:43:28,626 INFO HandlerThread:2030562 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:43:28,626 INFO SystemMonitor:2030562 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:43:28,626 INFO HandlerThread:2030562 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:43:28,627 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:43:28,627 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:43:28,628 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:43:28,628 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:43:28,629 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:43:28,639 DEBUG HandlerThread:2030562 [system_info.py:probe():152] Probing system +2025-01-04 21:43:28,641 DEBUG HandlerThread:2030562 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:probe():200] Probing system done +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:43:28.639636', 'startedAt': '2025-01-05T02:43:28.250380', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.54975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.018, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 700.6974601745605}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:43:28,645 INFO HandlerThread:2030562 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:43:28,645 INFO HandlerThread:2030562 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:_save_code():44] Saving code +2025-01-04 21:43:28,647 DEBUG HandlerThread:2030562 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:43:28,647 DEBUG HandlerThread:2030562 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:43:28,709 DEBUG HandlerThread:2030562 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:43:28,710 INFO HandlerThread:2030562 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:43:28,713 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:43:28,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:43:28,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:43:28,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:28,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:28,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:28,963 INFO HandlerThread:2030562 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/tb" + save: true +} +control { + relay_id: "c785e7d1d7ad44819dc4961b5cb38926" +} +_info { + stream_id: "97ao8n2y" +} + +2025-01-04 21:43:28,963 DEBUG HandlerThread:2030562 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:43:28,988 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/lyrdtuvr-diff.patch +2025-01-04 21:43:29,001 INFO wandb-upload_0:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/o34jqbai-wandb-metadata.json +2025-01-04 21:43:29,010 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/71m8c4y5-code/FISH/eval_robot.py +2025-01-04 21:43:29,050 DEBUG SenderThread:2030562 [sender.py:send():379] send: telemetry +2025-01-04 21:43:29,050 DEBUG SenderThread:2030562 [sender.py:send():379] send: tbrecord +2025-01-04 21:43:29,051 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:43:29,051 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 with policy live +2025-01-04 21:43:29,251 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/r667bvde-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH +2025-01-04 21:43:29,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:30,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:31,533 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:31,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:32,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:33,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:33,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:34,822 DEBUG SenderThread:2030562 [sender.py:send():379] send: config +2025-01-04 21:43:34,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:35,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:36,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:37,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:37,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:38,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:38,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:39,535 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:39,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:40,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:41,536 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:41,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:42,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:43,536 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:43,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:43,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:44,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:44,036 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:44,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:45,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:46,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:47,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:48,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:49,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:49,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:50,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:51,539 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:51,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:52,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:53,540 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:53,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:54,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:55,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:55,540 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:55,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:56,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:57,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:58,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:58,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:59,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:00,075 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:00,542 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:44:00,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:01,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:02,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:03,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:04,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:05,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:05,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:06,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:07,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:08,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:09,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:10,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:10,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:11,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:12,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:13,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:13,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:14,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:14,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:15,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:16,102 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:16,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:17,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:18,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:19,546 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:19,972 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:20,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:21,265 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:21,547 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:21,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:22,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:23,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:24,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:25,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:26,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:26,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:27,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:27,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:28,629 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:44:28,630 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:44:28,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:28,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:28,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:29,486 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:44:29,486 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 with policy now +2025-01-04 21:44:29,487 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:44:29,488 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:44:29,488 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:44:29,488 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:44:29,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:29,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media +2025-01-04 21:44:29,758 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/gvodhzjg-media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:29,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:30,026 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/cku9xji0-media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:30,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:31,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:31,972 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:32,496 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:32,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:33,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:34,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:35,551 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:35,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:36,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:37,552 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:37,857 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:37,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:38,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:39,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:40,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:41,553 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:44:41,801 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/gjvfn7nu-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:44:41,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:42,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:43,081 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:43,554 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:43,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:43,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:44,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:44,982 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:45,977 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:46,977 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:47,556 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:47,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:48,561 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:48,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:49,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:50,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:51,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:52,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:53,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:53,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:54,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:55,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:56,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:57,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:58,638 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:44:58,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:58,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:59,100 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:59,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:00,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:01,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:02,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:03,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:04,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:04,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:05,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:06,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:07,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:08,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:09,563 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:09,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:09,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:10,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:11,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:12,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:13,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:13,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:14,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:14,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:15,044 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:15,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:16,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:17,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:18,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:19,983 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:20,396 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:20,983 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:21,104 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:45:21,104 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 with policy now +2025-01-04 21:45:21,110 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:45:21,111 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:45:21,111 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:45:21,118 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:45:21,414 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/o1mkapio-media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:45:21,828 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/87xna3bd-media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:22,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:23,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:23,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:24,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:25,785 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:25,999 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:26,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:27,569 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:27,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:28,631 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:45:28,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:28,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:29,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:29,569 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:29,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:30,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:31,118 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:31,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:32,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:33,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:34,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:35,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:36,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:36,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:37,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:38,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:39,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:40,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:41,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:41,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:42,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:43,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:43,963 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:44,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:44,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:45,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:46,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:47,066 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:47,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:48,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:49,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:50,406 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:45:50,406 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:45:50,407 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:45:50,576 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:45:50,576 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:45:50,812 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/8szz4d48-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:45:50,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:51,577 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:51,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:52,793 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:52,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:53,577 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:53,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:54,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:55,578 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:55,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:56,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:57,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:57,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:58,631 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:45:58,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:58,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:59,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:00,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:01,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:02,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:03,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:03,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:04,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:05,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:06,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:07,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:08,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:08,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:09,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:10,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:11,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:12,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:13,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:13,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:14,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:14,098 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:14,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:15,998 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:16,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:17,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:18,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:19,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:19,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:20,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:21,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:22,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:23,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:24,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:24,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:25,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:26,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:27,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:28,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:46:28,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:28,970 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:29,000 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:29,345 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:46:29,345 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 with policy now +2025-01-04 21:46:29,346 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:46:29,347 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:46:29,347 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:46:29,349 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:46:29,589 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:29,589 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:46:29,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:29,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:46:29,617 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/ryvwbx2o-media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:29,886 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/jmnt44wo-media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:30,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:30,393 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:31,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:31,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:32,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:33,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:34,002 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:35,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:35,591 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:35,814 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:36,002 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:37,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:37,592 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:38,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:39,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:40,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:40,252 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:46:40,253 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:46:40,254 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:46:40,593 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:46:40,594 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:46:41,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:41,108 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:41,594 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:42,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:43,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:43,596 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:43,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:43,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:44,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:45,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:45,596 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:46,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:46,269 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:47,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:47,597 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:48,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:49,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:50,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:51,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:51,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:52,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:53,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:54,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:55,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:56,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:57,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:57,975 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:58,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:58,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:46:58,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:58,989 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:59,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:00,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:01,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:01,603 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:02,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:03,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:03,106 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:04,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:05,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:06,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:07,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:08,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:08,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:09,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:10,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:11,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:12,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:13,013 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:13,607 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:13,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:13,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:14,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:14,126 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:15,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:16,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:17,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:18,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:19,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:19,606 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:20,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:20,127 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:47:20,127 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 with policy now +2025-01-04 21:47:20,128 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:47:20,128 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:47:20,128 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:47:20,135 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:47:20,403 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/1fc6mkeh-media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:20,609 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:47:20,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:20,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:47:20,926 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/8v9msw1h-media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:21,027 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:21,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:22,025 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:23,026 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:24,026 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:24,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:25,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:25,611 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:26,032 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:27,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:27,612 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:28,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:28,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:47:28,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:28,966 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:29,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:29,329 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:47:29,329 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:47:29,330 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:47:29,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:47:29,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:47:29,845 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/sdjaoeeu-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:47:29,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:30,032 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:31,035 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:31,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:32,035 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:33,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:33,614 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:34,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:35,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:35,615 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:35,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:36,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:37,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:38,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:39,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:40,050 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:40,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:41,050 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:42,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:43,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:43,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:43,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:44,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:45,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:46,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:46,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:47,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:48,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:49,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:50,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:51,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:51,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:52,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:53,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:54,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:55,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:56,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:56,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:57,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:58,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:58,633 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:47:58,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:58,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:59,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:00,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:01,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:02,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:02,083 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:03,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:04,056 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:05,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:06,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:07,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:07,397 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:07,626 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:08,062 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:08,917 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:08,917 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 with policy now +2025-01-04 21:48:08,923 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:08,924 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:08,924 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:08,928 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:09,058 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:09,197 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/j0j3ff4f-media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:48:09,899 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/rtmuex3k-media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:10,073 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:11,063 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:11,628 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:12,080 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:12,585 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:13,061 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:13,628 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:13,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:48:13,981 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:48:14,067 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:15,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:16,076 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:17,081 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:17,630 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:18,079 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:18,441 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:19,084 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:20,083 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:21,082 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:22,088 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:23,098 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:23,986 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:24,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:25,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:26,113 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:27,108 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:28,095 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:28,633 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:48:28,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:48:28,979 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:48:29,067 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:29,100 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:29,403 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:29,403 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:29,403 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:29,634 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:29,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:29,738 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:48:29,738 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:48:29,878 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/5kdnzr06-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:29,945 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:30,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:30,214 INFO wandb-upload_1:2030562 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq0so4tw9 +2025-01-04 21:48:30,376 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:48:30,387 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:30,618 INFO SenderThread:2030562 [sender.py:send_request_log_artifact():1525] logged artifact run-97ao8n2y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE2NjAzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1ODQ0MA==', 'latestArtifact': None}} +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:30,619 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,619 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: telemetry +2025-01-04 21:48:30,620 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,623 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,624 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,624 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,624 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,624 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,625 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,625 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,626 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,626 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,626 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,626 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,627 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,627 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,627 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,627 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,628 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,628 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,628 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,628 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media +2025-01-04 21:48:30,636 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table +2025-01-04 21:48:30,636 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval +2025-01-04 21:48:30,823 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/ln0vfrd3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:30,845 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/dnmu79ca-media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:31,010 INFO wandb-upload_0:2030562 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3lal_vz3 +2025-01-04 21:48:31,061 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/q6m53ltv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:31,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:31,386 INFO SenderThread:2030562 [sender.py:send_request_log_artifact():1525] logged artifact run-97ao8n2y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE2NjA1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1ODQ0Mg==', 'latestArtifact': None}} +2025-01-04 21:48:31,386 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:31,386 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json with policy now +2025-01-04 21:48:31,387 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:31,387 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:31,387 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:31,629 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/qbolu01m-media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:31,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:31,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:32,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:33,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:34,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:34,985 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:35,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:36,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:37,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:38,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:39,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:39,421 DEBUG SenderThread:2030562 [sender.py:send():379] send: exit +2025-01-04 21:48:39,421 INFO SenderThread:2030562 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_exit():588] handling runtime: 310 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_exit():594] send defer +2025-01-04 21:48:39,422 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,422 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:48:39,422 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:48:39,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:48:39,423 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,423 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:48:39,423 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:48:39,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:48:39,423 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:48:39,423 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:48:39,424 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:48:39,424 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:48:39,497 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:48:39,497 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:48:39,498 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:48:39,498 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,498 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:48:39,498 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:48:39,498 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,498 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:48:39,499 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:48:39,499 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,499 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:48:39,499 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:48:39,499 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,499 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:48:39,639 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:41,639 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,411 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:48:46,411 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:46,412 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,412 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:48:46,412 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:46,413 DEBUG SenderThread:2030562 [sender.py:send():379] send: summary +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:46,413 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:48:46,413 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,413 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:48:46,414 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,414 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:48:46,417 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:46,525 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:48:46,525 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:46,525 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,525 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:48:46,526 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,526 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:48:46,641 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:46,642 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:48:47,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:49,155 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:48:49,155 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:49,156 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,156 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:48:49,156 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,156 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:48:49,156 INFO SenderThread:2030562 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:48:49,158 INFO SenderThread:2030562 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:48:49,160 INFO SenderThread:2030562 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:48:49,169 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:48:49,169 DEBUG SenderThread:2030562 [sender.py:send():379] send: artifact +2025-01-04 21:48:49,169 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,172 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:48:49,357 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:48:49,424 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml config.yaml +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json wandb-summary.json +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt requirements.txt +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:49,644 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log output.log +2025-01-04 21:48:49,644 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch diff.patch +2025-01-04 21:48:49,646 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:48:49,649 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:49,651 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,652 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:48:49,652 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,652 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:48:49,652 INFO SenderThread:2030562 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:48:49,833 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:48:49,835 INFO wandb-upload_0:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:49,848 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt +2025-01-04 21:48:49,903 INFO wandb-upload_4:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:49,922 INFO wandb-upload_3:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/djhbl8i3-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:50,122 INFO Thread-11 (_thread_body):2030562 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:48:50,122 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,122 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:48:50,122 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,123 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:48:50,123 INFO SenderThread:2030562 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:48:50,124 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,124 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:48:50,124 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:48:50,327 INFO SenderThread:2030562 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:48:50,327 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:48:50,327 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,328 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:48:50,328 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,328 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send():379] send: final +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send():379] send: footer +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:48:50,329 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:50,329 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:50,329 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:48:50,330 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:50,330 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:48:50,331 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:48:50,331 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:48:50,332 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:50,381 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:50,381 INFO MainThread:2030562 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:48:50,381 INFO MainThread:2030562 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:48:50,382 INFO MainThread:2030562 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:48:50,382 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:48:50,382 INFO HandlerThread:2030562 [handler.py:finish():882] shutting down handler +2025-01-04 21:48:51,331 INFO WriterThread:2030562 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [sender.py:finish():1615] shutting down sender +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [file_pusher.py:join():175] waiting for file pusher diff --git a/97ao8n2y/wandb/debug.log b/97ao8n2y/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09d4b95f50f65178febcbf4b3a8240e74959204a --- /dev/null +++ b/97ao8n2y/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Configure stats pid to 2030388 +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/settings +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug.log +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug-internal.log +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y'} +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():619] starting backend +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():623] setting up manager +2025-01-04 21:43:28,259 INFO MainThread:2030388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:43:28,262 INFO MainThread:2030388 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:43:28,274 INFO MainThread:2030388 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:43:28,280 INFO MainThread:2030388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:43:28,547 INFO MainThread:2030388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:43:28,615 INFO MainThread:2030388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:43:28,616 INFO MainThread:2030388 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:43:28,960 INFO MainThread:2030388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:43:28,963 INFO MainThread:2030388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:43:28,963 INFO MainThread:2030388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/tb, True +2025-01-04 21:43:34,821 INFO MainThread:2030388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:48:51,383 WARNING MsgRouterThr:2030388 [router.py:message_loop():77] message_loop has been closed diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43b18714d8fc9adc7a2d1e3ad66ad7826bcfc15e --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736045008 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f345052a4098779e6d1835354dda24725cd2d7e6 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..7b7e871 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4fefdef4398ca48a3dec922455e7dd712cf89ff6 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[40, 45, 0.8888888888888888, 0.7416145503037739, 0.9674066974777963, 0.7766186445398224, 0.9624643920927183, 0.796646214962281, 0.9469060077369471, 0.8080794549862298, 0.9434684588618258]]} \ No newline at end of file diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..234b21dc8bcc582b9c58fc101e014c4925eb06ef --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776ba6f8d1335d98898ca0dfb30b082dfb658f37cb805cc42a43084f746cad53 +size 1449040 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ee88c73c9192765bb0121660ac5170df5258ced --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6562e4bf5aa0b73485524dc80dce5b9e0cb31adde15a92cfb28904023cd41e32 +size 1464995 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b9e501c7d8ce6afff99a7cd0011e627bdf3e0ba3 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb26fcea183c3ae0aa38a4c46f5db42fa15333496efe601e273b9f1995f5e563 +size 1459054 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5be2aab9b9da2266f3fc5dd2c2e7dfcf1668350 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4651ece086df075e5442a24ab765a98e915558ed88e888a3c7088f53c4e12966 +size 1467788 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8f76203a0dba3d04c48c79f6fc505f2c835e80c0 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d451a7b8e4e09dc6abf67cb046aeee60f2e9a9dc12acf4e62f2182a9395df0f9 +size 1466610 diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f1630a990e8f81fb9160ef1f3a823c8a899fb162 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log @@ -0,0 +1,275 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736045014.822395]: resetting environment +[INFO] [1736045014.827247]: cleared current plan +[INFO] [1736045014.828667]: moving to home +[INFO] [1736045015.831044]: reached home +[INFO] [1736045015.832865]: reset action history +[INFO] [1736045018.543279]: environment reset +Starting episode 0 +[INFO] [1736045018.548758]: resetting environment +[INFO] [1736045018.556095]: cleared current plan +[INFO] [1736045018.557605]: moving to home +[INFO] [1736045019.560294]: reached home +[INFO] [1736045019.561949]: reset action history +[INFO] [1736045022.271517]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736045030.005721]: resetting environment +[INFO] [1736045030.010561]: cleared current plan +[INFO] [1736045030.011555]: moving to home +[INFO] [1736045031.013282]: reached home +[INFO] [1736045031.014568]: reset action history +[INFO] [1736045033.722851]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/episode_rosbags/episode_0_2025-01-04-21-43-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2030388.log +started roslaunch server http://158.130.50.37:35495/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2030668] +process[print_text-2]: started with pid [2030692] +process[rosbag_record-3]: started with pid [2030693] +started bagging! +For topic gripper_width: timestamp difference is -103657827 for nearest: 1736045056951541081 - target: 1736045057055198908 at idx 249 +For topic gripper_width: timestamp difference is 100487311 for nearest: 1736045059218210776 - target: 1736045059117723465 at idx 249 +For topic gripper_width: timestamp difference is 100074333 for nearest: 1736045064918210310 - target: 1736045064818135977 at idx 249 +[INFO] [1736045068.534971]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736045069.281738]: Stored episode 1. +[INFO] [1736045069.282059]: Saving video... +[INFO] [1736045069.471988]: Video saved! +Starting episode 1 +[INFO] [1736045069.487482]: resetting environment +[INFO] [1736045069.495373]: cleared current plan +[INFO] [1736045069.495679]: moving to home +[INFO] [1736045073.098727]: reached home +[INFO] [1736045073.099255]: reset action history +[INFO] [1736045075.809971]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736045081.032714]: resetting environment +[INFO] [1736045081.038715]: cleared current plan +[INFO] [1736045081.038915]: moving to home +[INFO] [1736045082.040481]: reached home +[INFO] [1736045082.040899]: reset action history +[INFO] [1736045084.747758]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/episode_rosbags/episode_1_2025-01-04-21-44-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2030388.log +started roslaunch server http://158.130.50.37:36891/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2030833] +process[print_text-5]: started with pid [2030834] +process[rosbag_record-6]: started with pid [2030835] +started bagging! +For topic gripper_width: timestamp difference is 117571352 for nearest: 1736045086518208979 - target: 1736045086400637627 at idx 27 +For topic gripper_width: timestamp difference is -165801020 for nearest: 1736045106851539640 - target: 1736045107017340660 at idx 249 +[INFO] [1736045119.614069]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736045120.395317]: Stored episode 2. +[INFO] [1736045120.396046]: Saving video... +[WARN] [1736045120.952611]: Plan exhausted +[WARN] [1736045120.982269]: Plan exhausted +[WARN] [1736045121.022496]: Plan exhausted +[WARN] [1736045121.062172]: Plan exhausted +[WARN] [1736045121.102031]: Plan exhausted +[INFO] [1736045121.071664]: Video saved! +Starting episode 2 +[INFO] [1736045121.115123]: resetting environment +[INFO] [1736045121.130357]: cleared current plan +[INFO] [1736045121.130584]: moving to home +[INFO] [1736045124.739197]: reached home +[INFO] [1736045124.741334]: reset action history +[INFO] [1736045127.525153]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736045149.730416]: resetting environment +[INFO] [1736045149.741517]: cleared current plan +[INFO] [1736045149.742773]: moving to home +[INFO] [1736045150.745374]: reached home +[INFO] [1736045150.747259]: reset action history +[INFO] [1736045153.455584]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/episode_rosbags/episode_2_2025-01-04-21-45-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2030388.log +started roslaunch server http://158.130.50.37:36059/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2030979] +process[print_text-8]: started with pid [2030980] +process[rosbag_record-9]: started with pid [2031004] +started bagging! +[INFO] [1736045188.309918]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736045189.036371]: Stored episode 3. +[INFO] [1736045189.044791]: Saving video... +[INFO] [1736045189.299177]: Video saved! +Starting episode 3 +[INFO] [1736045189.349372]: resetting environment +[INFO] [1736045189.388246]: cleared current plan +[INFO] [1736045189.392415]: moving to home +[INFO] [1736045192.999944]: reached home +[INFO] [1736045193.004911]: reset action history +[INFO] [1736045195.809497]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736045200.062776]: resetting environment +[INFO] [1736045200.095948]: cleared current plan +[INFO] [1736045200.099006]: moving to home +[INFO] [1736045201.105712]: reached home +[INFO] [1736045201.118512]: reset action history +[INFO] [1736045203.915639]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/episode_rosbags/episode_3_2025-01-04-21-46-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2030388.log +started roslaunch server http://158.130.50.37:34361/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2031134] +process[print_text-11]: started with pid [2031135] +process[rosbag_record-12]: started with pid [2031159] +started bagging! +For topic gripper_width: timestamp difference is 105752654 for nearest: 1736045205218228526 - target: 1736045205112475872 at idx 19 +For topic gripper_width: timestamp difference is -108858781 for nearest: 1736045218751537366 - target: 1736045218860396147 at idx 202 +For topic gripper_width: timestamp difference is 102172520 for nearest: 1736045232551541235 - target: 1736045232449368715 at idx 249 +[INFO] [1736045238.861960]: Storing episode... +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[INFO] [1736045239.603899]: Stored episode 4. +[INFO] [1736045239.609581]: Saving video... +[INFO] [1736045240.081645]: Video saved! +Starting episode 4 +[INFO] [1736045240.131649]: resetting environment +[INFO] [1736045240.171317]: cleared current plan +[INFO] [1736045240.171714]: moving to home +[INFO] [1736045243.784597]: reached home +[INFO] [1736045243.796607]: reset action history +[INFO] [1736045246.594336]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736045248.927065]: resetting environment +[INFO] [1736045248.946208]: cleared current plan +[INFO] [1736045248.951422]: moving to home +[INFO] [1736045249.953417]: reached home +[INFO] [1736045249.959631]: reset action history +[INFO] [1736045252.737776]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/episode_rosbags/episode_4_2025-01-04-21-47-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2030388.log +started roslaunch server http://158.130.50.37:34655/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2031285] +process[print_text-14]: started with pid [2031286] +process[rosbag_record-15]: started with pid [2031310] +started bagging! +For topic gripper_width: timestamp difference is 112342613 for nearest: 1736045286351537245 - target: 1736045286239194632 at idx 249 +[INFO] [1736045287.632451]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736045288.447974]: Stored episode 5. +[INFO] [1736045288.458633]: Saving video... +[INFO] [1736045288.869381]: Video saved! +[INFO] [1736045288.924749]: resetting environment +[INFO] [1736045288.961545]: cleared current plan +[INFO] [1736045288.965559]: moving to home +[INFO] [1736045292.584425]: reached home +[INFO] [1736045292.587657]: reset action history +[INFO] [1736045295.397519]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6c731770a29758381c4ecca73073463c9db3ddf8 --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:43:28.639636", + "startedAt": "2025-01-05T02:43:28.250380", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.54975, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.992, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.018, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 700.6974601745605 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..23b813d49314506734db1c5c18ebfc362c61539b --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "776ba6f8d1335d98898ca0dfb30b082dfb658f37cb805cc42a43084f746cad53", "size": 1449040, "path": "media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4"}, "global_step": 665, "_timestamp": 1736045310.2828248, "_runtime": 302.0196416378021, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6562e4bf5aa0b73485524dc80dce5b9e0cb31adde15a92cfb28904023cd41e32", "size": 1464995, "path": "media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "fb26fcea183c3ae0aa38a4c46f5db42fa15333496efe601e273b9f1995f5e563", "size": 1459054, "path": "media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4651ece086df075e5442a24ab765a98e915558ed88e888a3c7088f53c4e12966", "size": 1467788, "path": "media/videos/eval/3_eval_3_4651ece086df075e5442.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d451a7b8e4e09dc6abf67cb046aeee60f2e9a9dc12acf4e62f2182a9395df0f9", "size": 1466610, "path": "media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://xe7hr9th13id2j4yle0pir3712e2sruotyznmcy2prgah2cstjp3n4zfc7neh29dgllyb13wx1whyeqemw1pefmqr3kvzrlvof7g6gskz09guaextp1wrweuyoln79ri/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://x7pgmxxlabgiue0ubxk07vjonpxvkw9f7a4hldz3ve4n2s15v7mc6j30fnc7w9660d0d8a2o686uly56i5n710jq3ev6ocrw335xvlt76rkfr66g9n2ek3naw2cif02h:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 40, "total_num_episodes": 45, "total_success_rate": 0.8888888888888888, "total_success_rate_lb_0.01": 0.7416145503037739, "total_success_rate_ub_0.01": 0.9674066974777963, "total_success_rate_lb_0.025": 0.7766186445398224, "total_success_rate_ub_0.025": 0.9624643920927183, "total_success_rate_lb_0.05": 0.796646214962281, "total_success_rate_ub_0.05": 0.9469060077369471, "total_success_rate_lb_0.1": 0.8080794549862298, "total_success_rate_ub_0.1": 0.9434684588618258, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "05aa5e0c3562bd1e7e0b605a8a09d2573dc7364945f8d230ce9358dcd1d2853c", "size": 519, "artifact_path": "wandb-client-artifact://kvo9waasmr5g31pji655t8yqd6kw94btog5r3at975mv8wtt0rddia8n5u6sw9h8gbay4dhepp1vyzlmqp15x306cxv6fl5t5n8nv1t41haystf13ymajs9jccflomqj/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3apw413fmm2wcvtod3d1fgg963zozuu0u2lmrns08qfiv3101ya2wnb8ue3evq2dszpveeegndlx4r6i5k8f9byi0q7cjuaaszj52zdpd3bw140tz2284dpjrwadbbut:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 310}, "eval/total_success_rate": 0.8888888955116272, "eval/total_success_rate_lb_0.01": 0.7416145205497742, "eval/total_success_rate_ub_0.01": 0.967406690120697, "eval/total_success_rate_lb_0.025": 0.7766186594963074, "eval/total_success_rate_ub_0.025": 0.9624643921852112, "eval/total_success_rate_lb_0.05": 0.796646237373352, "eval/total_success_rate_ub_0.05": 0.9469060301780701, "eval/total_success_rate_lb_0.1": 0.8080794811248779, "eval/total_success_rate_ub_0.1": 0.943468451499939} \ No newline at end of file diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug-internal.log b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1c138e7395ee3f58db92b05f381c01a7f5fbb86d --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug-internal.log @@ -0,0 +1,833 @@ +2025-01-04 21:43:28,261 INFO StreamThr :2030562 [internal.py:wandb_internal():85] W&B internal server running at pid: 2030562, started at: 2025-01-04 21:43:28.260854 +2025-01-04 21:43:28,262 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:43:28,274 INFO WriterThread:2030562 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb +2025-01-04 21:43:28,274 DEBUG SenderThread:2030562 [sender.py:send():379] send: header +2025-01-04 21:43:28,286 DEBUG SenderThread:2030562 [sender.py:send():379] send: run +2025-01-04 21:43:28,532 INFO SenderThread:2030562 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files +2025-01-04 21:43:28,532 INFO SenderThread:2030562 [sender.py:_start_run_threads():1188] run started: 97ao8n2y with start time 1736045008.263183 +2025-01-04 21:43:28,547 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:43:28,547 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:43:28,621 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:43:28,626 DEBUG HandlerThread:2030562 [system_info.py:__init__():26] System info init +2025-01-04 21:43:28,626 DEBUG HandlerThread:2030562 [system_info.py:__init__():41] System info init done +2025-01-04 21:43:28,626 INFO HandlerThread:2030562 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:43:28,626 INFO SystemMonitor:2030562 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:43:28,626 INFO HandlerThread:2030562 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:43:28,627 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:43:28,627 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:43:28,628 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:43:28,628 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:43:28,629 INFO SystemMonitor:2030562 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:43:28,639 DEBUG HandlerThread:2030562 [system_info.py:probe():152] Probing system +2025-01-04 21:43:28,641 DEBUG HandlerThread:2030562 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:probe():200] Probing system done +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:43:28.639636', 'startedAt': '2025-01-05T02:43:28.250380', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.54975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.992, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.018, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 700.6974601745605}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:43:28,645 INFO HandlerThread:2030562 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:43:28,645 INFO HandlerThread:2030562 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:43:28,645 DEBUG HandlerThread:2030562 [system_info.py:_save_code():44] Saving code +2025-01-04 21:43:28,647 DEBUG HandlerThread:2030562 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:43:28,647 DEBUG HandlerThread:2030562 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:43:28,709 DEBUG HandlerThread:2030562 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:43:28,710 INFO HandlerThread:2030562 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:43:28,713 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:43:28,713 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:43:28,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:43:28,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:43:28,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:28,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:28,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:28,963 INFO HandlerThread:2030562 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/tb" + save: true +} +control { + relay_id: "c785e7d1d7ad44819dc4961b5cb38926" +} +_info { + stream_id: "97ao8n2y" +} + +2025-01-04 21:43:28,963 DEBUG HandlerThread:2030562 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:43:28,988 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/lyrdtuvr-diff.patch +2025-01-04 21:43:29,001 INFO wandb-upload_0:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/o34jqbai-wandb-metadata.json +2025-01-04 21:43:29,010 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/71m8c4y5-code/FISH/eval_robot.py +2025-01-04 21:43:29,050 DEBUG SenderThread:2030562 [sender.py:send():379] send: telemetry +2025-01-04 21:43:29,050 DEBUG SenderThread:2030562 [sender.py:send():379] send: tbrecord +2025-01-04 21:43:29,051 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:43:29,051 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 with policy live +2025-01-04 21:43:29,251 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/r667bvde-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code +2025-01-04 21:43:29,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH +2025-01-04 21:43:29,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:30,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:31,533 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:31,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:32,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:33,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:33,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:34,822 DEBUG SenderThread:2030562 [sender.py:send():379] send: config +2025-01-04 21:43:34,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:35,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:36,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:37,534 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:37,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:38,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:38,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:39,535 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:39,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:40,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:41,536 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:41,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:42,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:43,536 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:43,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:43,960 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:44,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:44,036 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:44,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:45,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:46,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:47,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:48,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:49,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:49,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:50,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:51,539 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:51,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:52,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:53,540 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:53,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:54,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:55,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:43:55,540 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:43:55,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:56,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:57,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:58,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:43:58,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:43:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:43:59,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:00,075 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:00,542 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:44:00,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:01,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:02,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:03,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:04,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:05,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:05,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:06,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:07,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:08,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:09,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:10,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:10,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:11,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:12,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:13,960 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:13,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:14,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:14,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:15,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:16,102 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:16,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:17,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:18,969 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:19,546 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:19,972 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:20,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:21,265 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:21,547 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:21,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:22,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:23,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:24,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:25,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:26,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:26,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:27,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:27,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:28,629 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:44:28,630 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:44:28,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:28,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:28,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:29,486 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:44:29,486 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 with policy now +2025-01-04 21:44:29,487 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:44:29,488 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:44:29,488 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:44:29,488 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:44:29,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:29,549 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:44:29,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media +2025-01-04 21:44:29,758 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/gvodhzjg-media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:29,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:30,026 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/cku9xji0-media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:44:30,971 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:31,550 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:31,972 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:32,496 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:32,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:33,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:34,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:35,551 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:35,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:36,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:37,552 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:37,857 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:37,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:38,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:39,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:40,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:41,553 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:44:41,801 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/gjvfn7nu-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:44:41,973 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:42,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:43,081 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:43,554 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:43,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:43,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:44,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:44,982 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:45,977 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:46,977 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:47,556 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:44:47,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:48,561 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:48,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:49,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:50,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:51,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:52,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:53,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:53,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:54,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:55,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:56,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:57,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:58,638 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:44:58,961 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:44:58,961 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:44:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:44:59,100 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:44:59,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:00,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:01,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:02,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:03,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:04,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:04,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:05,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:06,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:07,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:08,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:09,563 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:09,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:09,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:10,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:11,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:12,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:13,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:13,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:14,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:14,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:15,044 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:15,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:16,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:17,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:18,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:19,983 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:20,396 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:20,983 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:21,104 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:45:21,104 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 with policy now +2025-01-04 21:45:21,110 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:45:21,111 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:45:21,111 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:45:21,118 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:45:21,414 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/o1mkapio-media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:45:21,828 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/87xna3bd-media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:45:21,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:22,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:23,567 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:23,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:24,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:25,785 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:25,999 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:26,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:27,569 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:27,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:28,631 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:45:28,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:28,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:29,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:29,569 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:29,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:30,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:31,118 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:31,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:32,988 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:33,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:34,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:35,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:36,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:36,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:37,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:38,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:39,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:40,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:41,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:41,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:42,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:43,963 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:43,963 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:44,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:44,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:45,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:46,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:47,066 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:47,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:48,990 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:49,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:50,406 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:45:50,406 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:45:50,407 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:45:50,576 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:45:50,576 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:45:50,812 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/8szz4d48-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:45:50,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:51,577 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:51,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:52,793 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:52,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:53,577 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:53,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:54,991 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:55,578 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:45:55,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:56,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:57,968 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:45:57,992 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:58,631 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:45:58,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:45:58,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:45:59,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:45:59,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:00,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:01,993 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:02,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:03,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:03,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:04,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:05,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:06,994 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:07,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:08,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:08,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:09,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:10,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:11,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:12,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:13,962 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:13,962 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:14,005 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:14,098 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:14,995 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:15,998 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:16,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:17,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:18,996 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:19,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:19,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:20,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:21,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:22,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:23,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:24,967 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:24,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:25,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:26,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:27,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:28,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:46:28,970 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:28,970 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:29,000 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:29,345 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:46:29,345 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 with policy now +2025-01-04 21:46:29,346 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:46:29,347 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:46:29,347 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:46:29,349 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:46:29,589 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:29,589 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:46:29,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:29,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:46:29,617 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/ryvwbx2o-media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:29,886 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/jmnt44wo-media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:46:30,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:30,393 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:31,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:31,590 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:32,001 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:33,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:34,002 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:35,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:35,591 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:35,814 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:36,002 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:37,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:37,592 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:38,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:39,004 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:40,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:40,252 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:46:40,253 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:46:40,254 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:46:40,593 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:46:40,594 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:46:41,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:41,108 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:41,594 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:42,009 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:43,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:43,596 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:43,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:43,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:44,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:45,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:45,596 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:46,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:46,269 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:47,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:47,597 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:46:48,006 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:49,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:50,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:51,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:51,974 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:52,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:53,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:54,007 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:55,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:56,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:57,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:57,975 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:46:58,008 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:46:58,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:46:58,989 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:46:58,989 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:46:59,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:00,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:01,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:01,603 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:02,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:03,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:03,106 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:04,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:05,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:06,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:07,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:08,010 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:08,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:09,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:10,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:11,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:12,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:13,013 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:13,607 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:13,964 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:13,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:14,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:14,126 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:15,011 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:16,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:17,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:18,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:19,012 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:19,606 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:20,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:20,127 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:47:20,127 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 with policy now +2025-01-04 21:47:20,128 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:47:20,128 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:47:20,128 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:47:20,135 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:47:20,403 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/1fc6mkeh-media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:20,609 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:47:20,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:20,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:47:20,926 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/8v9msw1h-media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:47:21,027 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:21,610 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:22,025 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:23,026 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:24,026 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:24,987 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:25,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:25,611 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:26,032 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:27,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:27,612 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:28,029 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:28,632 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:47:28,966 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:28,966 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:29,028 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:29,329 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:47:29,329 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:47:29,330 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:47:29,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:47:29,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:47:29,845 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/sdjaoeeu-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:47:29,997 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:30,032 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:31,035 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:31,613 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:32,035 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:33,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:33,614 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:34,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:35,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:35,615 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:47:35,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:36,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:37,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:38,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:39,049 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:40,050 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:40,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:41,050 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:42,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:43,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:43,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:43,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:44,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:45,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:46,051 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:46,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:47,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:48,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:49,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:50,052 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:51,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:51,979 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:52,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:53,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:54,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:55,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:56,053 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:56,980 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:47:57,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:58,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:47:58,633 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:47:58,965 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:47:58,965 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:47:59,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:00,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:01,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:02,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:02,083 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:03,054 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:04,056 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:05,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:06,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:07,055 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:07,397 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:07,626 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:08,062 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:08,917 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:08,917 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 with policy now +2025-01-04 21:48:08,923 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:08,924 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:08,924 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:08,928 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:09,058 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:09,197 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/j0j3ff4f-media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:09,627 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval +2025-01-04 21:48:09,899 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/rtmuex3k-media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:10,073 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:11,063 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:11,628 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:12,080 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:12,585 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:13,061 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:13,628 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:13,981 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:48:13,981 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:48:14,067 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:15,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:16,076 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:17,081 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:17,630 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:18,079 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:18,441 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:19,084 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:20,083 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:21,082 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:22,088 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:23,098 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:23,986 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:24,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:25,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:26,113 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:27,108 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:28,095 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:28,633 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:48:28,978 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:48:28,979 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:48:29,067 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:29,100 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:29,403 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:29,403 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:29,403 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:29,634 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:29,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:29,738 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:48:29,738 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:48:29,878 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/5kdnzr06-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:29,945 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:30,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:30,214 INFO wandb-upload_1:2030562 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq0so4tw9 +2025-01-04 21:48:30,376 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:48:30,387 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:48:30,618 INFO SenderThread:2030562 [sender.py:send_request_log_artifact():1525] logged artifact run-97ao8n2y-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE2NjAzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1ODQ0MA==', 'latestArtifact': None}} +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:30,619 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,619 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,619 DEBUG SenderThread:2030562 [sender.py:send():379] send: telemetry +2025-01-04 21:48:30,620 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,623 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,624 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,624 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,624 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,624 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,625 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,625 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,625 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,626 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,626 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,626 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,626 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,627 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,627 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,627 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,627 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,628 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,628 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:30,628 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:30,628 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:30,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media +2025-01-04 21:48:30,636 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table +2025-01-04 21:48:30,636 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval +2025-01-04 21:48:30,823 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/ln0vfrd3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:30,845 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/dnmu79ca-media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:31,010 INFO wandb-upload_0:2030562 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3lal_vz3 +2025-01-04 21:48:31,061 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/q6m53ltv-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:31,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:31,386 INFO SenderThread:2030562 [sender.py:send_request_log_artifact():1525] logged artifact run-97ao8n2y-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE2NjA1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1ODQ0Mg==', 'latestArtifact': None}} +2025-01-04 21:48:31,386 DEBUG SenderThread:2030562 [sender.py:send():379] send: files +2025-01-04 21:48:31,386 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json with policy now +2025-01-04 21:48:31,387 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:31,387 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:31,387 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:31,629 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/qbolu01m-media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:31,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:31,635 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:32,091 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:33,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:34,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:34,985 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:35,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:36,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:37,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:38,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:39,092 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:39,421 DEBUG SenderThread:2030562 [sender.py:send():379] send: exit +2025-01-04 21:48:39,421 INFO SenderThread:2030562 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_exit():588] handling runtime: 310 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_exit():594] send defer +2025-01-04 21:48:39,422 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,422 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:48:39,422 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:48:39,422 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:48:39,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:48:39,423 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,423 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:48:39,423 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:48:39,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:48:39,423 INFO HandlerThread:2030562 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:48:39,423 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:48:39,423 DEBUG SystemMonitor:2030562 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:48:39,424 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:48:39,424 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:48:39,497 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:48:39,497 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:48:39,498 INFO HandlerThread:2030562 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:48:39,498 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,498 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:48:39,498 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:48:39,498 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,498 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:48:39,499 DEBUG SenderThread:2030562 [sender.py:send():379] send: stats +2025-01-04 21:48:39,499 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:39,499 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:48:39,499 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:48:39,499 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:39,499 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:48:39,639 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:41,639 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:48:46,410 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,411 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:48:46,411 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send():379] send: history +2025-01-04 21:48:46,411 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:48:46,412 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,412 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:48:46,412 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:46,413 DEBUG SenderThread:2030562 [sender.py:send():379] send: summary +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:48:46,413 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:48:46,413 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:48:46,413 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,413 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:48:46,414 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,414 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:48:46,417 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:48:46,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:46,525 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:48:46,525 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:46,525 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:46,525 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:48:46,526 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:46,526 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:48:46,641 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:46,642 INFO Thread-12 :2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:48:47,423 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:49,155 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:48:49,155 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:49,156 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,156 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:48:49,156 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,156 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:48:49,156 INFO SenderThread:2030562 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:48:49,158 INFO SenderThread:2030562 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:48:49,160 INFO SenderThread:2030562 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:48:49,169 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:48:49,169 DEBUG SenderThread:2030562 [sender.py:send():379] send: artifact +2025-01-04 21:48:49,169 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,172 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:48:49,357 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:48:49,357 INFO SenderThread:2030562 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:48:49,424 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml config.yaml +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json wandb-summary.json +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt requirements.txt +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:48:49,643 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:49,644 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log output.log +2025-01-04 21:48:49,644 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/diff.patch diff.patch +2025-01-04 21:48:49,646 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 media/videos/eval/4_eval_4_d451a7b8e4e09dc6abf6.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 media/videos/eval/2_eval_2_fb26fcea183c3ae0aa38.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 media/videos/eval/3_eval_3_4651ece086df075e5442.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 media/videos/eval/1_eval_1_6562e4bf5aa0b7348552.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 media/videos/eval/0_eval_0_776ba6f8d1335d98898c.mp4 +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json media/table/eval/total_success_rate_ci_6_05aa5e0c3562bd1e7e0b.table.json +2025-01-04 21:48:49,649 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:48:49,649 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:49,651 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:49,652 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:48:49,652 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:49,652 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:48:49,652 INFO SenderThread:2030562 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:48:49,833 INFO wandb-upload_2:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/config.yaml +2025-01-04 21:48:49,835 INFO wandb-upload_0:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/wandb-summary.json +2025-01-04 21:48:49,848 INFO wandb-upload_1:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/requirements.txt +2025-01-04 21:48:49,903 INFO wandb-upload_4:2030562 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/files/output.log +2025-01-04 21:48:49,922 INFO wandb-upload_3:2030562 [upload_job.py:push():130] Uploaded file /tmp/tmp1ea_cgsfwandb/djhbl8i3-events.out.tfevents.1736045008.leonmkim-ROG-Strix-G15CS-G15CS.2030388.0 +2025-01-04 21:48:50,122 INFO Thread-11 (_thread_body):2030562 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:48:50,122 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,122 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:48:50,122 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,123 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:48:50,123 INFO SenderThread:2030562 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:48:50,124 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,124 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:48:50,124 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:48:50,124 INFO SenderThread:2030562 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:48:50,327 INFO SenderThread:2030562 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:48:50,327 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:48:50,327 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,328 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:48:50,328 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:48:50,328 INFO HandlerThread:2030562 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send():379] send: final +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send():379] send: footer +2025-01-04 21:48:50,328 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: defer +2025-01-04 21:48:50,328 INFO SenderThread:2030562 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:48:50,329 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:50,329 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:50,329 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:48:50,330 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:48:50,330 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:48:50,331 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:48:50,331 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:48:50,332 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:48:50,381 DEBUG SenderThread:2030562 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:48:50,381 INFO MainThread:2030562 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:48:50,381 INFO MainThread:2030562 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:48:50,382 INFO MainThread:2030562 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:48:50,382 DEBUG HandlerThread:2030562 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:48:50,382 INFO HandlerThread:2030562 [handler.py:finish():882] shutting down handler +2025-01-04 21:48:51,331 INFO WriterThread:2030562 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [sender.py:finish():1615] shutting down sender +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:48:51,381 INFO SenderThread:2030562 [file_pusher.py:join():175] waiting for file pusher diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug.log b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09d4b95f50f65178febcbf4b3a8240e74959204a --- /dev/null +++ b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Configure stats pid to 2030388 +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/settings +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:43:28,253 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug.log +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/logs/debug-internal.log +2025-01-04 21:43:28,254 INFO MainThread:2030388 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y'} +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():619] starting backend +2025-01-04 21:43:28,255 INFO MainThread:2030388 [wandb_init.py:init():623] setting up manager +2025-01-04 21:43:28,259 INFO MainThread:2030388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:43:28,262 INFO MainThread:2030388 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:43:28,274 INFO MainThread:2030388 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:43:28,280 INFO MainThread:2030388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:43:28,547 INFO MainThread:2030388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:43:28,615 INFO MainThread:2030388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:43:28,616 INFO MainThread:2030388 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:43:28,960 INFO MainThread:2030388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:43:28,961 INFO MainThread:2030388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:43:28,963 INFO MainThread:2030388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:43:28,963 INFO MainThread:2030388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/97ao8n2y/tb, True +2025-01-04 21:43:34,821 INFO MainThread:2030388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:48:51,383 WARNING MsgRouterThr:2030388 [router.py:message_loop():77] message_loop has been closed diff --git a/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3dfe4b46b9927d751fc9fe6a16b81cb15bd06a77 Binary files /dev/null and b/97ao8n2y/wandb/run-20250104_214328-97ao8n2y/run-97ao8n2y.wandb differ diff --git a/9xaoecg6/.hydra/config.yaml b/9xaoecg6/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/9xaoecg6/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/9xaoecg6/.hydra/hydra.yaml b/9xaoecg6/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a901325246c9a826eaa680dc8dec0b5163dacd5 --- /dev/null +++ b/9xaoecg6/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/9xaoecg6 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/9xaoecg6/.hydra/overrides.yaml b/9xaoecg6/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/9xaoecg6/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/9xaoecg6/eval_policy.log b/9xaoecg6/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..3905235fe571c599ef1069821096051759275df1 --- /dev/null +++ b/9xaoecg6/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 20:45:53,883][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:45:53,886][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 20:45:54,959][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/br14p9rb/.hydra/config.yaml b/br14p9rb/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/br14p9rb/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/br14p9rb/.hydra/hydra.yaml b/br14p9rb/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc0ba907857661838d233206c522673e6ab9e314 --- /dev/null +++ b/br14p9rb/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/br14p9rb/.hydra/overrides.yaml b/br14p9rb/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/br14p9rb/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/br14p9rb/episode_rosbags/aligned_depth_to_color_K.npy b/br14p9rb/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/br14p9rb/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/br14p9rb/episode_rosbags/cam_tf_world.npy b/br14p9rb/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/br14p9rb/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/br14p9rb/episode_rosbags/color_K.npy b/br14p9rb/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/br14p9rb/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/br14p9rb/episode_rosbags/depth_K.npy b/br14p9rb/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/br14p9rb/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/br14p9rb/eval_robot.log b/br14p9rb/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6ba25154bc16ac00b597b8b103d726b64f5a933b --- /dev/null +++ b/br14p9rb/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:11:56,588][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:11:56,590][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/br14p9rb/eval_video/0_eval.mp4 b/br14p9rb/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..351c20f477f5468deaeeb7ad6f2e7d9f69706a46 --- /dev/null +++ b/br14p9rb/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92869d54d5ca1018f4a9bb6852ed81ab7edc10ae5c340a89af50554da2a06345 +size 1502269 diff --git a/br14p9rb/eval_video/1_eval.mp4 b/br14p9rb/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7608ff5126ee88394bcaf1f628f98c6932b0d077 --- /dev/null +++ b/br14p9rb/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363a77cf1a9e04df339566acf4bf724254c368afa1ccd227cc3365debe22c19c +size 1521497 diff --git a/br14p9rb/eval_video/2_eval.mp4 b/br14p9rb/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e55242b45f22460b5512fb36185c715108b85f7 --- /dev/null +++ b/br14p9rb/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e1974516cae1f65fc3b742c70d9b85ff110e3ae59f0400b401569b88ae450f +size 1514367 diff --git a/br14p9rb/eval_video/3_eval.mp4 b/br14p9rb/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b48c0acf38c26dbdbded338f53816e4e6edd7196 --- /dev/null +++ b/br14p9rb/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:129e9d1f1ab8c01c94d06da1f957be66107387bddd77e7f4dca227622ea41fb7 +size 1504619 diff --git a/br14p9rb/eval_video/4_eval.mp4 b/br14p9rb/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67568dd9c45cb3ee8549743d97da716004db5f03 --- /dev/null +++ b/br14p9rb/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4792f0829cf66c6b9d8adb4cb6e95482f58f8772163b6b161a767fa25df30909 +size 1501060 diff --git a/br14p9rb/tb/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 b/br14p9rb/tb/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 new file mode 100644 index 0000000000000000000000000000000000000000..87d4a9fa663d45e20d36b5c951ca14a13c044d7a --- /dev/null +++ b/br14p9rb/tb/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae57e8af8aa948d5c4443dbbbaf017f0c46e1a256bbd53ed39a5938fbc6b7ca0 +size 6539 diff --git a/br14p9rb/wandb/debug-internal.log b/br14p9rb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6eec222a6e24ebb700c569ebd1358b5a4c421c43 --- /dev/null +++ b/br14p9rb/wandb/debug-internal.log @@ -0,0 +1,1079 @@ +2025-01-04 22:12:02,381 INFO StreamThr :2033938 [internal.py:wandb_internal():85] W&B internal server running at pid: 2033938, started at: 2025-01-04 22:12:02.381017 +2025-01-04 22:12:02,382 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:12:02,394 INFO WriterThread:2033938 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb +2025-01-04 22:12:02,394 DEBUG SenderThread:2033938 [sender.py:send():379] send: header +2025-01-04 22:12:02,405 DEBUG SenderThread:2033938 [sender.py:send():379] send: run +2025-01-04 22:12:02,654 INFO SenderThread:2033938 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files +2025-01-04 22:12:02,654 INFO SenderThread:2033938 [sender.py:_start_run_threads():1188] run started: br14p9rb with start time 1736046722.382562 +2025-01-04 22:12:02,678 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:12:02,678 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:12:02,765 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:12:02,770 DEBUG HandlerThread:2033938 [system_info.py:__init__():26] System info init +2025-01-04 22:12:02,770 DEBUG HandlerThread:2033938 [system_info.py:__init__():41] System info init done +2025-01-04 22:12:02,770 INFO HandlerThread:2033938 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:12:02,770 INFO SystemMonitor:2033938 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:12:02,770 INFO HandlerThread:2033938 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:12:02,771 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:12:02,771 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:12:02,772 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:12:02,772 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:12:02,773 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:12:02,783 DEBUG HandlerThread:2033938 [system_info.py:probe():152] Probing system +2025-01-04 22:12:02,785 DEBUG HandlerThread:2033938 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:probe():200] Probing system done +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:12:02.783618', 'startedAt': '2025-01-05T03:12:02.374537', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.2435, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9206237792969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:12:02,789 INFO HandlerThread:2033938 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:12:02,789 INFO HandlerThread:2033938 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:_save_code():44] Saving code +2025-01-04 22:12:02,791 DEBUG HandlerThread:2033938 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:12:02,791 DEBUG HandlerThread:2033938 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:12:02,853 DEBUG HandlerThread:2033938 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:12:02,853 INFO HandlerThread:2033938 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:12:02,856 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:12:02,856 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:12:02,857 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:12:02,857 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:12:03,101 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:12:03,101 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:12:03,103 DEBUG SenderThread:2033938 [sender.py:send():379] send: telemetry +2025-01-04 22:12:03,110 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/qq9qy3pd-wandb-metadata.json +2025-01-04 22:12:03,117 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/yxlcxvwt-diff.patch +2025-01-04 22:12:03,131 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/r7lkriq0-code/FISH/eval_robot.py +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:03,145 INFO HandlerThread:2033938 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/tb" + save: true +} +control { + relay_id: "ef5843e7f8e54450bffab8b34651ed96" +} +_info { + stream_id: "br14p9rb" +} + +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:12:03,146 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:03,201 DEBUG SenderThread:2033938 [sender.py:send():379] send: tbrecord +2025-01-04 22:12:03,202 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:12:03,202 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 with policy live +2025-01-04 22:12:03,413 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/bllycg0h-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH +2025-01-04 22:12:03,657 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code +2025-01-04 22:12:04,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:05,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:05,655 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:06,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:07,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:08,063 DEBUG SenderThread:2033938 [sender.py:send():379] send: config +2025-01-04 22:12:08,063 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:08,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:09,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:09,657 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:10,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:11,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:11,658 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:12,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:13,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:13,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:13,659 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:14,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:15,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:15,660 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:16,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:17,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:17,660 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:18,101 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:18,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:18,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:18,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:19,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:20,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:21,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:22,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:23,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:24,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:24,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:25,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:26,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:27,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:27,663 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:28,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:29,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:29,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:29,664 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:30,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:31,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:31,664 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:32,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:33,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:33,106 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:33,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:34,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:34,261 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:34,665 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:12:35,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:36,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:37,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:38,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:39,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:40,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:40,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:41,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:41,667 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:42,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:43,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:44,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:45,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:45,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:45,668 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:46,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:47,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:47,669 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:48,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:48,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:48,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:49,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:50,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:50,248 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:51,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:52,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:53,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:54,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:55,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:56,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:56,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:57,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:58,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:59,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:00,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:01,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:01,269 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:01,672 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:02,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:02,773 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:13:02,774 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:13:03,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:03,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:03,673 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:04,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:05,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:05,854 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:13:05,854 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 with policy now +2025-01-04 22:13:05,854 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:13:05,855 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:13:05,855 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:13:05,856 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:13:06,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:06,140 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/3gzygjhq-media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media +2025-01-04 22:13:06,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:13:06,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos +2025-01-04 22:13:06,864 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:06,938 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fem9d0y3-media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:07,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:07,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:08,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:09,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:09,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:10,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:11,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:11,676 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:12,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:12,521 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:13,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:13,677 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:14,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:15,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:16,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:17,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:18,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:18,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:18,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:18,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:19,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:20,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:21,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:22,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:23,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:23,358 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:23,680 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:13:23,909 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/lor7etkm-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:13:24,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:25,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:25,681 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:26,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:27,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:28,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:29,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:29,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:29,682 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:30,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:31,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:32,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:32,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:13:33,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:33,103 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:33,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:34,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:34,186 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:35,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:36,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:37,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:38,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:39,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:40,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:40,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:41,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:42,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:43,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:44,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:45,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:45,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:46,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:47,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:48,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:48,103 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:48,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:49,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:50,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:50,247 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:51,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:52,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:53,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:53,690 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:54,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:55,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:56,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:56,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:57,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:58,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:59,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:00,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:01,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:01,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:02,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:02,775 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:14:03,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:03,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:03,124 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:03,301 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:14:03,301 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:14:03,302 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 with policy now +2025-01-04 22:14:03,302 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:03,302 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:03,328 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:03,560 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fznmglo3-media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:03,693 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:03,693 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:03,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:03,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:14:03,946 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/no3h86jb-media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:04,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:05,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:05,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:06,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:06,329 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:07,136 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:07,695 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:08,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:09,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:10,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:11,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:11,505 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:11,696 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:12,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:13,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:13,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:14:14,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:14,572 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:14,572 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:14,573 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:14,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:15,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:15,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:16,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:16,661 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:17,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:18,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:18,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:18,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:19,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:19,699 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:20,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:21,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:22,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:22,260 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:23,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:24,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:25,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:26,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:27,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:28,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:28,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:29,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:30,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:31,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:32,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:32,775 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:14:33,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:33,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:33,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:33,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:34,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:35,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:36,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:37,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:38,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:39,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:39,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:40,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:41,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:42,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:43,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:44,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:44,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:45,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:46,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:47,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:48,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:48,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:48,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:49,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:49,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:50,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:51,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:52,136 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:53,140 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:53,281 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:14:53,281 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:14:53,282 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 with policy now +2025-01-04 22:14:53,282 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:53,282 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:53,285 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:53,534 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/1sg2i1ej-media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:14:53,968 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/24etsvzs-media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:54,142 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:54,308 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:55,147 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:55,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:56,147 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:57,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:57,711 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:58,150 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:59,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:59,406 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:00,148 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:01,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:01,713 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:02,146 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:02,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:15:03,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:03,114 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:03,157 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:04,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:05,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:05,193 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:06,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:07,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:08,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:09,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:10,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:11,123 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:11,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:12,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:13,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:14,156 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:15,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:16,157 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:17,122 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:17,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:18,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:18,114 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:18,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:19,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:20,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:21,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:22,166 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:22,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:23,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:24,160 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:25,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:26,162 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:27,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:28,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:28,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:29,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:30,160 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:31,164 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:32,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:32,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:15:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:33,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:33,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:33,174 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:34,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:35,167 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:36,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:37,166 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:38,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:39,122 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:39,172 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:40,169 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:41,174 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:42,179 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:43,175 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:44,177 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:45,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:45,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:46,178 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:47,176 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:47,461 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:15:47,462 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:15:47,463 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:15:47,728 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:15:47,729 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:15:47,966 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/95u33p4l-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:15:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:48,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:48,178 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:49,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:49,729 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:50,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:50,501 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:51,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:52,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:53,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:53,730 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:54,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:55,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:56,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:56,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:57,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:58,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:59,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:00,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:01,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:01,389 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:01,733 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:02,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:02,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:16:03,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:03,116 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:03,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:04,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:05,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:05,735 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:06,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:07,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:07,285 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:08,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:09,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:10,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:11,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:11,737 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:12,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:12,505 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:13,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:14,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:15,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:15,739 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:16,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:17,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:17,657 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:18,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:18,115 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:18,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:19,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:20,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:21,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:22,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:23,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:23,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:24,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:25,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:26,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:27,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:27,365 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:16:27,365 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:16:27,365 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 with policy now +2025-01-04 22:16:27,366 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:16:27,366 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:16:27,366 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:16:27,625 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/htmi0rq3-media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:16:28,016 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fnli2a9z-media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:28,123 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:28,190 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:29,186 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:29,744 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:30,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:31,190 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:31,744 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:32,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:32,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:16:33,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:33,120 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:33,189 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:33,253 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:34,194 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:35,195 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:35,746 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:36,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:37,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:38,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:39,124 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:39,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:40,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:41,203 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:42,204 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:43,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:44,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:44,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:45,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:46,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:47,206 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:48,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:48,120 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:48,204 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:49,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:49,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:50,206 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:51,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:52,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:53,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:54,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:55,126 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:55,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:56,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:57,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:58,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:58,560 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:16:58,560 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:16:58,561 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:16:58,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:16:58,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:16:58,982 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/9scxdb02-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:16:59,220 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:59,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:00,218 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:00,941 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:01,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:02,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:02,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:17:03,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:03,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:03,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:03,757 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:04,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:05,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:06,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:06,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:07,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:08,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:09,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:09,759 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:10,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:11,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:12,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:12,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:13,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:14,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:15,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:16,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:17,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:17,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:18,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:18,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:18,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:19,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:20,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:21,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:22,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:22,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:23,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:24,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:25,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:25,764 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:26,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:27,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:28,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:28,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:29,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:29,766 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:30,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:31,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:32,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:32,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:17:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:33,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:33,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:33,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:34,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:35,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:36,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:37,221 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:37,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:37,912 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:37,912 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 with policy now +2025-01-04 22:17:37,918 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:37,919 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:37,919 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:37,922 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:38,175 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/l3qx37z0-media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:38,217 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:38,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:38,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:38,771 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:17:39,063 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/0z6dseik-media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:39,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:39,221 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:39,771 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:40,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:41,224 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:42,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:43,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:43,772 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:44,138 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:44,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:45,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:45,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:46,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:46,618 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:46,618 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:46,619 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:46,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:46,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:17:47,026 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:17:47,026 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:17:47,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:47,306 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:47,449 INFO wandb-upload_0:2033938 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk9l0lb5b +2025-01-04 22:17:47,706 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:17:47,718 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:47,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval +2025-01-04 22:17:47,794 INFO SenderThread:2033938 [sender.py:send_request_log_artifact():1525] logged artifact run-br14p9rb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4NTYwOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTkwOQ==', 'latestArtifact': None}} +2025-01-04 22:17:47,794 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:47,794 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-04 22:17:47,795 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:47,795 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,798 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,798 DEBUG SenderThread:2033938 [sender.py:send():379] send: telemetry +2025-01-04 22:17:47,798 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,799 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,799 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,799 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,799 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,800 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,800 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,801 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,801 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,801 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,801 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,802 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,802 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,803 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,803 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,804 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,804 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:17:47,978 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/tdeyyxvo-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:47,981 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/kqlpoxzf-media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:48,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:48,167 INFO wandb-upload_0:2033938 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpol9ft8ro +2025-01-04 22:17:48,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:48,306 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/jpbnk4yl-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:48,528 INFO SenderThread:2033938 [sender.py:send_request_log_artifact():1525] logged artifact run-br14p9rb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4NTYyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTkxNA==', 'latestArtifact': None}} +2025-01-04 22:17:48,528 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:48,528 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json with policy now +2025-01-04 22:17:48,528 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:48,529 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:48,529 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:48,529 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:48,759 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/xgtg4wb1-media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:48,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:49,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:49,666 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:49,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:50,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:51,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:52,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:53,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:54,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:55,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:55,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:56,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:57,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:58,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:59,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:00,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:00,247 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:00,779 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:18:01,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:02,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:02,778 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:18:03,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:03,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:03,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:04,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:05,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:05,276 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:06,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:07,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:08,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:09,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:10,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:11,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:11,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:12,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:13,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:14,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:15,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:16,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:16,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:17,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:18,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:18,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:18,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:19,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:20,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:21,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:21,275 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:22,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:23,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:24,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:25,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:26,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:27,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:27,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:28,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:29,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:30,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:31,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:32,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:32,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:32,779 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:18:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:33,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:33,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:34,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:35,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:36,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:37,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:37,263 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:38,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:39,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:40,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:41,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:42,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:43,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:43,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:44,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:45,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:46,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:47,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:48,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:48,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:48,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:49,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:50,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:51,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:52,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:53,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:54,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:54,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:55,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:56,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:57,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:58,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:59,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:59,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:00,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:01,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:02,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:02,779 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:19:03,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:03,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:03,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:04,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:04,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:05,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:06,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:07,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:08,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:09,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:10,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:10,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:11,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:12,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:13,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:14,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:15,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:15,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:16,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:17,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:18,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:18,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:18,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:19,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:20,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:20,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:21,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:22,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:23,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:24,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:25,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:26,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:26,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:27,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:28,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:29,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:30,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:31,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:32,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:32,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:32,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:19:33,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:33,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:33,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:34,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:35,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:36,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:37,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:37,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:38,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:39,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:40,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:41,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:42,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:43,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:43,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:44,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:45,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:46,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:47,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:48,119 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:48,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:48,249 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:49,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:50,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:51,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:52,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:53,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:54,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:54,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:55,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:56,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:57,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:58,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:59,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:59,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:00,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:01,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:02,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:02,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:20:03,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:20:03,119 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:20:03,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:04,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:04,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:05,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:06,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:07,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:08,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:09,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:10,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:10,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:11,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:12,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:13,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:14,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:15,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:15,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:15,329 DEBUG SenderThread:2033938 [sender.py:send():379] send: exit +2025-01-04 22:20:15,329 INFO SenderThread:2033938 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_exit():588] handling runtime: 492 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_exit():594] send defer +2025-01-04 22:20:15,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,330 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:20:15,330 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:20:15,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:20:15,331 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,331 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:20:15,331 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:20:15,331 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:20:15,331 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:20:15,331 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:20:15,332 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:20:15,332 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:20:15,383 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:20:15,384 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:20:15,384 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:20:15,384 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,384 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:20:15,384 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:20:15,384 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:20:15,385 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,385 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:20:15,385 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,385 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:20:15,385 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:20:15,385 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,385 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:20:15,835 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:17,836 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:20:21,645 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,645 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:20:21,645 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:20:21,645 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:21,646 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:20:21,646 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:20:21,646 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:21,646 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,646 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:20:21,647 DEBUG SenderThread:2033938 [sender.py:send():379] send: summary +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:21,647 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,648 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:20:21,648 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,648 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:20:21,648 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,649 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:20:21,649 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,649 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:20:21,753 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:20:21,753 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,753 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:20:21,753 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,753 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:20:21,754 INFO SenderThread:2033938 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:20:21,756 INFO SenderThread:2033938 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:20:21,758 INFO SenderThread:2033938 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:20:21,769 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:20:21,769 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,769 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:20:21,769 DEBUG SenderThread:2033938 [sender.py:send():379] send: artifact +2025-01-04 22:20:21,838 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:21,838 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:20:21,938 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:20:22,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml config.yaml +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json wandb-summary.json +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt requirements.txt +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:20:22,840 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log output.log +2025-01-04 22:20:22,843 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch diff.patch +2025-01-04 22:20:22,845 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:20:22,849 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:22,849 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:22,849 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:20:22,849 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:20:23,011 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:23,030 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:20:23,083 INFO wandb-upload_3:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt +2025-01-04 22:20:23,083 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/qt0vmmye-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:20:23,104 INFO wandb-upload_4:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:23,304 INFO Thread-11 (_thread_body):2033938 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:20:23,304 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,304 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:20:23,305 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,305 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:20:23,305 INFO SenderThread:2033938 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:20:23,306 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,306 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:20:23,306 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:20:23,331 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,578 INFO SenderThread:2033938 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:20:23,578 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:20:23,578 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,578 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,578 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send():379] send: final +2025-01-04 22:20:23,579 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send():379] send: footer +2025-01-04 22:20:23,579 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:20:23,581 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,581 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:23,581 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:20:23,646 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,646 INFO MainThread:2033938 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:20:23,646 INFO MainThread:2033938 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:20:23,647 INFO MainThread:2033938 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:20:23,647 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:20:23,647 INFO HandlerThread:2033938 [handler.py:finish():882] shutting down handler +2025-01-04 22:20:24,581 INFO WriterThread:2033938 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [sender.py:finish():1615] shutting down sender +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [file_pusher.py:join():175] waiting for file pusher diff --git a/br14p9rb/wandb/debug.log b/br14p9rb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..79efe5bbe7d02b45fb6529b726a72a16d0dddc5d --- /dev/null +++ b/br14p9rb/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Configure stats pid to 2033763 +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/settings +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug.log +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug-internal.log +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb'} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():619] starting backend +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():623] setting up manager +2025-01-04 22:12:02,380 INFO MainThread:2033763 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:12:02,381 INFO MainThread:2033763 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:12:02,393 INFO MainThread:2033763 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:12:02,402 INFO MainThread:2033763 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:12:02,677 INFO MainThread:2033763 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:12:02,717 INFO MainThread:2033763 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:12:02,717 INFO MainThread:2033763 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:12:03,103 INFO MainThread:2033763 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:12:03,104 INFO MainThread:2033763 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:12:03,104 INFO MainThread:2033763 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/tb, True +2025-01-04 22:12:08,062 INFO MainThread:2033763 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:20:24,647 WARNING MsgRouterThr:2033763 [router.py:message_loop():77] message_loop has been closed diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db7e845e34cce3fc49c95c175e82ee28ed23937d --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736046722 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f345052a4098779e6d1835354dda24725cd2d7e6 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..7b7e871 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json new file mode 100644 index 0000000000000000000000000000000000000000..1e458a3799b28b692b354a26fa4018a18462926e --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[46, 60, 0.7666666666666667, 0.6214599013085633, 0.8761350511752307, 0.6513147204811733, 0.8655769526227312, 0.6706842109219597, 0.8446120321061417, 0.6852890327202243, 0.8355424124076429]]} \ No newline at end of file diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..351c20f477f5468deaeeb7ad6f2e7d9f69706a46 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92869d54d5ca1018f4a9bb6852ed81ab7edc10ae5c340a89af50554da2a06345 +size 1502269 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7608ff5126ee88394bcaf1f628f98c6932b0d077 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363a77cf1a9e04df339566acf4bf724254c368afa1ccd227cc3365debe22c19c +size 1521497 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9e55242b45f22460b5512fb36185c715108b85f7 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10e1974516cae1f65fc3b742c70d9b85ff110e3ae59f0400b401569b88ae450f +size 1514367 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b48c0acf38c26dbdbded338f53816e4e6edd7196 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:129e9d1f1ab8c01c94d06da1f957be66107387bddd77e7f4dca227622ea41fb7 +size 1504619 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67568dd9c45cb3ee8549743d97da716004db5f03 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4792f0829cf66c6b9d8adb4cb6e95482f58f8772163b6b161a767fa25df30909 +size 1501060 diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..87a1925f7cb6cae27a4d5b96398059440b1c86fd --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736046728.063703]: resetting environment +[INFO] [1736046728.070254]: cleared current plan +[INFO] [1736046728.071162]: moving to home +[INFO] [1736046729.073383]: reached home +[INFO] [1736046729.074579]: reset action history +[INFO] [1736046731.784463]: environment reset +Starting episode 0 +[INFO] [1736046731.790128]: resetting environment +[INFO] [1736046731.797864]: cleared current plan +[INFO] [1736046731.798819]: moving to home +[INFO] [1736046732.801148]: reached home +[INFO] [1736046732.802385]: reset action history +[INFO] [1736046735.511698]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736046746.255063]: resetting environment +[INFO] [1736046746.259061]: cleared current plan +[INFO] [1736046746.260144]: moving to home +[INFO] [1736046747.262338]: reached home +[INFO] [1736046747.263566]: reset action history +[INFO] [1736046749.971446]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/episode_rosbags/episode_0_2025-01-04-22-12-29.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2033763.log +started roslaunch server http://158.130.50.37:45475/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2034044] +process[print_text-2]: started with pid [2034068] +process[rosbag_record-3]: started with pid [2034069] +started bagging! +For topic gripper_width: timestamp difference is 103029236 for nearest: 1736046760418212876 - target: 1736046760315183640 at idx 137 +For topic gripper_width: timestamp difference is 105081527 for nearest: 1736046763018186061 - target: 1736046762913104534 at idx 171 +For topic gripper_width: timestamp difference is -325344733 for nearest: 1736046766251494714 - target: 1736046766576839447 at idx 218 +For topic gripper_width: timestamp difference is 118564004 for nearest: 1736046779718188876 - target: 1736046779599624872 at idx 249 +For topic gripper_width: timestamp difference is 118967814 for nearest: 1736046780218188090 - target: 1736046780099220276 at idx 249 +For topic gripper_width: timestamp difference is 119970725 for nearest: 1736046781518185542 - target: 1736046781398214817 at idx 249 +[INFO] [1736046784.898196]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736046785.627512]: Stored episode 1. +[INFO] [1736046785.628047]: Saving video... +[INFO] [1736046785.839593]: Video saved! +Starting episode 1 +[INFO] [1736046785.854865]: resetting environment +[INFO] [1736046785.863995]: cleared current plan +[INFO] [1736046785.864279]: moving to home +[INFO] [1736046788.766095]: reached home +[INFO] [1736046788.766507]: reset action history +[INFO] [1736046791.479098]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736046803.363108]: resetting environment +[INFO] [1736046803.368173]: cleared current plan +[INFO] [1736046803.368337]: moving to home +[INFO] [1736046804.369225]: reached home +[INFO] [1736046804.369661]: reset action history +[INFO] [1736046807.078430]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/episode_rosbags/episode_1_2025-01-04-22-13-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2033763.log +started roslaunch server http://158.130.50.37:34469/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2034199] +process[print_text-5]: started with pid [2034200] +process[rosbag_record-6]: started with pid [2034224] +started bagging! +For topic gripper_width: timestamp difference is 118695880 for nearest: 1736046831518153573 - target: 1736046831399457693 at idx 249 +[INFO] [1736046841.992651]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736046842.742450]: Stored episode 2. +[INFO] [1736046842.743161]: Saving video... +[INFO] [1736046843.259609]: Video saved! +Starting episode 2 +[INFO] [1736046843.285994]: resetting environment +[INFO] [1736046843.310399]: cleared current plan +[INFO] [1736046843.312456]: moving to home +[INFO] [1736046846.718406]: reached home +[INFO] [1736046846.721592]: reset action history +[INFO] [1736046849.459058]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736046853.607225]: resetting environment +[INFO] [1736046853.614608]: cleared current plan +[INFO] [1736046853.614953]: moving to home +[INFO] [1736046854.616578]: reached home +[INFO] [1736046854.617008]: reset action history +[INFO] [1736046857.325755]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/episode_rosbags/episode_2_2025-01-04-22-14-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2033763.log +started roslaunch server http://158.130.50.37:33585/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2034348] +process[print_text-8]: started with pid [2034349] +process[rosbag_record-9]: started with pid [2034350] +started bagging! +[INFO] [1736046892.064445]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736046892.895594]: Stored episode 3. +[INFO] [1736046892.898723]: Saving video... +[INFO] [1736046893.237125]: Video saved! +Starting episode 3 +[INFO] [1736046893.282274]: resetting environment +[INFO] [1736046893.307253]: cleared current plan +[INFO] [1736046893.307499]: moving to home +[INFO] [1736046896.613314]: reached home +[INFO] [1736046896.625067]: reset action history +[INFO] [1736046899.405721]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736046947.431374]: resetting environment +[INFO] [1736046947.451536]: cleared current plan +[INFO] [1736046947.453315]: moving to home +[INFO] [1736046948.456806]: reached home +[INFO] [1736046948.460639]: reset action history +[INFO] [1736046951.244416]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/episode_rosbags/episode_3_2025-01-04-22-15-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2033763.log +started roslaunch server http://158.130.50.37:37419/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2034520] +process[print_text-11]: started with pid [2034521] +process[rosbag_record-12]: started with pid [2034522] +started bagging! +For topic gripper_width: timestamp difference is -102193258 for nearest: 1736046960151515581 - target: 1736046960253708839 at idx 125 +For topic gripper_width: timestamp difference is 100911402 for nearest: 1736046964218135618 - target: 1736046964117224216 at idx 170 +For topic gripper_width: timestamp difference is 105880471 for nearest: 1736046970451477261 - target: 1736046970345596790 at idx 247 +For topic gripper_width: timestamp difference is 109347577 for nearest: 1736046974618187423 - target: 1736046974508839846 at idx 249 +[INFO] [1736046986.265448]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736046987.002645]: Stored episode 4. +[INFO] [1736046987.008952]: Saving video... +[INFO] [1736046987.319710]: Video saved! +Starting episode 4 +[INFO] [1736046987.366852]: resetting environment +[INFO] [1736046987.388903]: cleared current plan +[INFO] [1736046987.393923]: moving to home +[INFO] [1736046990.700710]: reached home +[INFO] [1736046990.708066]: reset action history +[INFO] [1736046993.495534]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047017.874561]: resetting environment +[INFO] [1736047017.895944]: cleared current plan +[INFO] [1736047017.897278]: moving to home +[INFO] [1736047018.899673]: reached home +[INFO] [1736047018.904331]: reset action history +[INFO] [1736047021.634279]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/episode_rosbags/episode_4_2025-01-04-22-17-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2033763.log +started roslaunch server http://158.130.50.37:45553/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2034690] +process[print_text-14]: started with pid [2034691] +process[rosbag_record-15]: started with pid [2034692] +started bagging! +For topic gripper_width: timestamp difference is 115281196 for nearest: 1736047027818184228 - target: 1736047027702903032 at idx 91 +For topic gripper_width: timestamp difference is 103648585 for nearest: 1736047044518181008 - target: 1736047044414532423 at idx 249 +For topic gripper_width: timestamp difference is 101215216 for nearest: 1736047047118184420 - target: 1736047047016969204 at idx 249 +For topic gripper_width: timestamp difference is 108822380 for nearest: 1736047056518186365 - target: 1736047056409363985 at idx 249 +[INFO] [1736047056.750130]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736047057.557705]: Stored episode 5. +[INFO] [1736047057.560152]: Saving video... +[INFO] [1736047057.824654]: Video saved! +[INFO] [1736047057.922530]: resetting environment +[INFO] [1736047057.951777]: cleared current plan +[INFO] [1736047057.952029]: moving to home +[INFO] [1736047061.458618]: reached home +[INFO] [1736047061.471183]: reset action history +[INFO] [1736047064.296902]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5399ac2421a27c8721fa4b20f07c5f66030789b9 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:12:02.783618", + "startedAt": "2025-01-05T03:12:02.374537", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.2435, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.982, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 736.9206237792969 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f32137336d4d9486c7c1c070704402ca0013dc5f --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "92869d54d5ca1018f4a9bb6852ed81ab7edc10ae5c340a89af50554da2a06345", "size": 1502269, "path": "media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4"}, "global_step": 665, "_timestamp": 1736047067.6324558, "_runtime": 345.2498939037323, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "363a77cf1a9e04df339566acf4bf724254c368afa1ccd227cc3365debe22c19c", "size": 1521497, "path": "media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4"}, "eval/num_success": 1.0, "episode": 60.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "10e1974516cae1f65fc3b742c70d9b85ff110e3ae59f0400b401569b88ae450f", "size": 1514367, "path": "media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "129e9d1f1ab8c01c94d06da1f957be66107387bddd77e7f4dca227622ea41fb7", "size": 1504619, "path": "media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4792f0829cf66c6b9d8adb4cb6e95482f58f8772163b6b161a767fa25df30909", "size": 1501060, "path": "media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://wysjo2i14zmjuvs063f625opt6cwi6hq4un829o62337h0zd17qdiohccvlfecnu8c5zhjdgijrxil69eean4qhthjwr92sdqna7arjjecullupt277kyu1woridu2mn/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://31ekxisko6bn8k82wgx0pm96ny96itpai301qv3n869hrfbqualr6c41qfo8ck6ymdr1wv5dyzx2zkk6wq0gvoqd9gsw942hvx3va8tui4r2koubgprbdirfqk33mks4:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 46, "total_num_episodes": 60, "total_success_rate": 0.7666666666666667, "total_success_rate_lb_0.01": 0.6214599013085633, "total_success_rate_ub_0.01": 0.8761350511752307, "total_success_rate_lb_0.025": 0.6513147204811733, "total_success_rate_ub_0.025": 0.8655769526227312, "total_success_rate_lb_0.05": 0.6706842109219597, "total_success_rate_ub_0.05": 0.8446120321061417, "total_success_rate_lb_0.1": 0.6852890327202243, "total_success_rate_ub_0.1": 0.8355424124076429, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f6c18c2ba742daffddcb27d534af66fcc3076456725379741b446e5b5d37ce36", "size": 520, "artifact_path": "wandb-client-artifact://zi2mzwp69aukujtijnfzqav75zl3pbgmsunnao9tbsq7l7wnnspmkymyy8wyd9pn2zfyxsd0zsh2ipc1hz82bxdgslfoeqx80uq7eh7vugwkrpk32cxe17vxwmiswbri/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r5y1xixyrr5515rbfyhdtk29rx9ju2r9y5sofgk70kbbf2rxbcgpcjo3v0ync3p9hpm6dn05iriso3wfi1iarboqre271gbw2x0hsvehl6ww85l6mw6g9oipwd2n832w:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 492}, "eval/total_success_rate": 0.7666666507720947, "eval/total_success_rate_lb_0.01": 0.6214599013328552, "eval/total_success_rate_ub_0.01": 0.8761350512504578, "eval/total_success_rate_lb_0.025": 0.6513147354125977, "eval/total_success_rate_ub_0.025": 0.8655769228935242, "eval/total_success_rate_lb_0.05": 0.6706842184066772, "eval/total_success_rate_ub_0.05": 0.8446120023727417, "eval/total_success_rate_lb_0.1": 0.6852890253067017, "eval/total_success_rate_ub_0.1": 0.8355424404144287} \ No newline at end of file diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug-internal.log b/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6eec222a6e24ebb700c569ebd1358b5a4c421c43 --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug-internal.log @@ -0,0 +1,1079 @@ +2025-01-04 22:12:02,381 INFO StreamThr :2033938 [internal.py:wandb_internal():85] W&B internal server running at pid: 2033938, started at: 2025-01-04 22:12:02.381017 +2025-01-04 22:12:02,382 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:12:02,394 INFO WriterThread:2033938 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb +2025-01-04 22:12:02,394 DEBUG SenderThread:2033938 [sender.py:send():379] send: header +2025-01-04 22:12:02,405 DEBUG SenderThread:2033938 [sender.py:send():379] send: run +2025-01-04 22:12:02,654 INFO SenderThread:2033938 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files +2025-01-04 22:12:02,654 INFO SenderThread:2033938 [sender.py:_start_run_threads():1188] run started: br14p9rb with start time 1736046722.382562 +2025-01-04 22:12:02,678 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:12:02,678 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:12:02,765 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:12:02,770 DEBUG HandlerThread:2033938 [system_info.py:__init__():26] System info init +2025-01-04 22:12:02,770 DEBUG HandlerThread:2033938 [system_info.py:__init__():41] System info init done +2025-01-04 22:12:02,770 INFO HandlerThread:2033938 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:12:02,770 INFO SystemMonitor:2033938 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:12:02,770 INFO HandlerThread:2033938 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:12:02,771 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:12:02,771 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:12:02,772 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:12:02,772 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:12:02,773 INFO SystemMonitor:2033938 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:12:02,783 DEBUG HandlerThread:2033938 [system_info.py:probe():152] Probing system +2025-01-04 22:12:02,785 DEBUG HandlerThread:2033938 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:probe():200] Probing system done +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:12:02.783618', 'startedAt': '2025-01-05T03:12:02.374537', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.2435, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9206237792969}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:12:02,789 INFO HandlerThread:2033938 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:12:02,789 INFO HandlerThread:2033938 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:12:02,789 DEBUG HandlerThread:2033938 [system_info.py:_save_code():44] Saving code +2025-01-04 22:12:02,791 DEBUG HandlerThread:2033938 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:12:02,791 DEBUG HandlerThread:2033938 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:12:02,853 DEBUG HandlerThread:2033938 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:12:02,853 INFO HandlerThread:2033938 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:12:02,856 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:12:02,856 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:12:02,857 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:12:02,857 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:12:03,101 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:12:03,101 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:12:03,103 DEBUG SenderThread:2033938 [sender.py:send():379] send: telemetry +2025-01-04 22:12:03,110 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/qq9qy3pd-wandb-metadata.json +2025-01-04 22:12:03,117 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/yxlcxvwt-diff.patch +2025-01-04 22:12:03,131 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/r7lkriq0-code/FISH/eval_robot.py +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:03,145 INFO HandlerThread:2033938 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/tb" + save: true +} +control { + relay_id: "ef5843e7f8e54450bffab8b34651ed96" +} +_info { + stream_id: "br14p9rb" +} + +2025-01-04 22:12:03,145 DEBUG HandlerThread:2033938 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:12:03,146 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:03,201 DEBUG SenderThread:2033938 [sender.py:send():379] send: tbrecord +2025-01-04 22:12:03,202 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:12:03,202 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 with policy live +2025-01-04 22:12:03,413 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/bllycg0h-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:12:03,656 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH +2025-01-04 22:12:03,657 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code +2025-01-04 22:12:04,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:05,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:05,655 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:06,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:07,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:08,063 DEBUG SenderThread:2033938 [sender.py:send():379] send: config +2025-01-04 22:12:08,063 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:08,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:09,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:09,657 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:10,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:11,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:11,658 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:12,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:13,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:13,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:13,659 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:14,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:15,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:15,660 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:16,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:17,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:17,660 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:18,101 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:18,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:18,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:18,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:19,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:20,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:21,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:22,104 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:23,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:24,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:24,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:25,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:26,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:27,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:27,663 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:28,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:29,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:29,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:29,664 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:30,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:31,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:31,664 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:32,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:33,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:33,106 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:33,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:34,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:34,261 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:34,665 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:12:35,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:36,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:37,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:38,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:39,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:40,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:40,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:41,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:41,667 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:42,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:43,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:44,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:45,105 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:45,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:45,668 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:46,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:47,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:47,669 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:12:48,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:12:48,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:12:48,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:49,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:50,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:50,248 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:51,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:52,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:53,109 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:54,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:55,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:56,106 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:12:56,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:57,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:58,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:12:59,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:00,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:01,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:01,269 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:01,672 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:02,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:02,773 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:13:02,774 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:13:03,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:03,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:03,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:03,673 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:04,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:05,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:05,854 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:13:05,854 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 with policy now +2025-01-04 22:13:05,854 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:13:05,855 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:13:05,855 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:13:05,856 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:13:06,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:06,140 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/3gzygjhq-media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:06,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media +2025-01-04 22:13:06,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:13:06,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos +2025-01-04 22:13:06,864 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:06,938 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fem9d0y3-media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:13:07,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:07,674 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:08,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:09,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:09,675 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:10,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:11,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:11,676 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:12,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:12,521 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:13,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:13,677 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:14,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:15,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:16,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:17,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:18,102 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:18,102 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:18,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:18,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:19,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:20,112 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:21,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:22,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:23,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:23,358 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:23,680 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:13:23,909 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/lor7etkm-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:13:24,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:25,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:25,681 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:26,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:27,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:28,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:29,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:29,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:29,682 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:30,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:31,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:32,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:32,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:13:33,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:33,103 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:33,145 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:34,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:34,186 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:35,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:36,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:37,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:38,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:39,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:40,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:40,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:41,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:42,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:43,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:44,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:45,107 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:45,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:46,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:47,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:48,103 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:13:48,103 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:13:48,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:49,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:50,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:50,247 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:51,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:52,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:53,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:53,690 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:13:54,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:55,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:56,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:13:56,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:57,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:58,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:13:59,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:00,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:01,108 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:01,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:02,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:02,775 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:14:03,110 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:03,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:03,124 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:03,301 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:14:03,301 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:14:03,302 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 with policy now +2025-01-04 22:14:03,302 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:03,302 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:03,328 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:03,560 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fznmglo3-media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:03,693 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:03,693 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:03,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:03,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:14:03,946 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/no3h86jb-media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:14:04,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:05,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:05,694 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:06,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:06,329 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:07,136 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:07,695 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:08,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:09,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:10,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:11,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:11,505 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:11,696 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:12,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:13,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:13,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:14:14,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:14,572 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:14,572 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:14,573 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:14,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:15,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:15,697 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:16,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:16,661 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:17,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:18,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:18,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:18,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:19,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:19,699 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:20,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:21,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:22,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:22,260 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:23,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:24,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:25,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:26,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:27,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:28,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:28,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:29,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:30,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:31,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:32,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:32,775 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:14:33,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:33,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:33,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:33,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:34,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:35,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:36,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:37,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:38,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:39,114 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:39,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:40,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:41,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:42,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:43,134 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:44,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:44,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:45,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:46,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:47,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:48,111 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:14:48,111 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:14:48,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:49,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:49,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:50,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:51,135 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:52,136 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:53,140 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:53,281 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:14:53,281 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:14:53,282 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 with policy now +2025-01-04 22:14:53,282 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:14:53,282 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:14:53,285 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:14:53,534 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/1sg2i1ej-media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:53,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:14:53,968 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/24etsvzs-media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:14:54,142 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:54,308 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:14:55,147 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:55,710 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:56,147 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:57,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:57,711 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:14:58,150 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:59,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:14:59,406 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:00,148 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:01,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:01,713 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:02,146 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:02,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:15:03,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:03,114 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:03,157 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:04,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:05,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:05,193 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:06,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:07,153 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:08,149 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:09,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:10,152 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:11,123 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:11,151 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:12,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:13,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:14,156 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:15,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:16,157 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:17,122 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:17,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:18,113 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:18,114 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:18,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:19,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:20,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:21,155 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:22,166 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:22,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:23,158 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:24,160 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:25,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:26,162 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:27,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:28,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:28,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:29,161 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:30,160 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:31,164 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:32,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:32,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:15:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:33,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:33,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:33,174 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:34,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:35,167 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:36,165 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:37,166 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:38,168 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:39,122 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:39,172 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:40,169 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:41,174 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:42,179 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:43,175 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:44,177 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:45,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:45,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:46,178 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:47,176 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:47,461 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:15:47,462 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:15:47,463 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:15:47,728 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:15:47,729 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:15:47,966 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/95u33p4l-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:15:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:15:48,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:15:48,178 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:49,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:49,729 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:50,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:50,501 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:51,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:52,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:53,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:53,730 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:15:54,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:55,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:56,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:15:56,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:57,180 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:58,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:15:59,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:00,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:01,181 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:01,389 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:01,733 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:02,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:02,776 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:16:03,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:03,116 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:03,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:04,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:05,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:05,735 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:06,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:07,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:07,285 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:08,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:09,182 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:10,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:11,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:11,737 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:12,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:12,505 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:13,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:14,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:15,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:15,739 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:16,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:17,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:17,657 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:18,115 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:18,115 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:18,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:19,183 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:20,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:21,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:22,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:23,121 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:23,184 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:24,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:25,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:26,185 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:27,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:27,365 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:16:27,365 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:16:27,365 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 with policy now +2025-01-04 22:16:27,366 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:16:27,366 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:16:27,366 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:16:27,625 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/htmi0rq3-media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:27,743 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:16:28,016 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/fnli2a9z-media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:16:28,123 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:28,190 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:29,186 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:29,744 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:30,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:31,190 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:31,744 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:32,187 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:32,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:16:33,120 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:33,120 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:33,189 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:33,253 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:34,194 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:35,195 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:35,746 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:16:36,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:37,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:38,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:39,124 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:39,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:40,200 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:41,203 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:42,204 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:43,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:44,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:44,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:45,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:46,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:47,206 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:48,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:16:48,120 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:16:48,204 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:49,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:49,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:50,206 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:51,205 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:52,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:53,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:54,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:55,126 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:16:55,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:56,208 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:57,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:58,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:58,560 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:16:58,560 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:16:58,561 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:16:58,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:16:58,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:16:58,982 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/9scxdb02-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:16:59,220 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:16:59,755 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:00,218 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:00,941 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:01,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:02,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:02,777 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:17:03,116 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:03,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:03,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:03,757 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:04,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:05,211 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:06,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:06,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:07,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:08,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:09,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:09,759 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:10,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:11,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:12,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:12,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:13,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:14,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:15,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:16,212 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:17,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:17,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:18,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:18,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:18,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:19,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:20,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:21,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:22,198 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:22,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:23,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:24,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:25,213 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:25,764 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:26,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:27,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:28,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:28,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:29,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:29,766 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:30,214 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:31,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:32,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:32,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:17:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:33,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:33,209 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:33,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:34,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:35,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:36,215 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:37,221 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:37,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:37,912 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:37,912 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 with policy now +2025-01-04 22:17:37,918 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:37,919 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:37,919 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:37,922 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:38,175 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/l3qx37z0-media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:38,217 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:38,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:38,770 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:38,771 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval +2025-01-04 22:17:39,063 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/0z6dseik-media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:17:39,125 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:39,221 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:39,771 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:40,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:41,224 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:42,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:43,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:43,772 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:44,138 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:44,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:45,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:45,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:46,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:46,618 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:46,618 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:46,619 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:46,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:46,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:17:47,026 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:17:47,026 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:17:47,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:47,306 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:47,449 INFO wandb-upload_0:2033938 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk9l0lb5b +2025-01-04 22:17:47,706 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:17:47,718 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:17:47,773 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table +2025-01-04 22:17:47,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval +2025-01-04 22:17:47,794 INFO SenderThread:2033938 [sender.py:send_request_log_artifact():1525] logged artifact run-br14p9rb-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4NTYwOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTkwOQ==', 'latestArtifact': None}} +2025-01-04 22:17:47,794 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:47,794 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-04 22:17:47,795 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:47,795 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,798 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,798 DEBUG SenderThread:2033938 [sender.py:send():379] send: telemetry +2025-01-04 22:17:47,798 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,799 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,799 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,799 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,799 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,800 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,800 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,801 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,801 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,801 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,801 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,802 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,802 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,802 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,803 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,803 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,803 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:47,804 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:47,804 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:17:47,978 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/tdeyyxvo-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:47,981 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/kqlpoxzf-media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:48,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:17:48,167 INFO wandb-upload_0:2033938 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpol9ft8ro +2025-01-04 22:17:48,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:48,306 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/jpbnk4yl-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:17:48,528 INFO SenderThread:2033938 [sender.py:send_request_log_artifact():1525] logged artifact run-br14p9rb-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE4NTYyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MTkxNA==', 'latestArtifact': None}} +2025-01-04 22:17:48,528 DEBUG SenderThread:2033938 [sender.py:send():379] send: files +2025-01-04 22:17:48,528 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json with policy now +2025-01-04 22:17:48,528 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:17:48,529 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:17:48,529 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:17:48,529 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:17:48,759 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/xgtg4wb1-media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:17:48,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:17:49,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:49,666 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:49,774 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:17:50,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:51,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:52,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:53,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:54,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:55,129 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:17:55,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:56,225 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:57,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:58,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:17:59,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:00,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:00,247 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:00,779 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:18:01,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:02,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:02,778 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:18:03,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:03,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:03,226 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:04,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:05,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:05,276 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:06,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:07,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:08,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:09,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:10,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:11,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:11,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:12,227 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:13,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:14,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:15,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:16,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:16,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:17,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:18,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:18,117 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:18,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:19,228 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:20,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:21,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:21,275 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:22,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:23,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:24,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:25,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:26,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:27,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:27,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:28,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:29,229 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:30,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:31,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:32,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:32,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:32,779 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:18:33,117 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:33,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:33,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:34,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:35,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:36,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:37,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:37,263 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:38,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:39,230 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:40,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:41,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:42,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:43,130 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:43,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:44,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:45,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:46,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:47,231 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:18:48,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:18:48,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:48,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:49,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:50,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:51,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:52,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:53,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:54,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:54,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:55,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:56,232 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:57,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:58,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:18:59,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:18:59,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:00,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:01,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:02,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:02,779 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:19:03,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:03,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:03,233 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:04,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:04,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:05,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:06,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:07,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:08,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:09,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:10,131 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:10,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:11,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:12,234 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:13,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:14,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:15,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:15,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:16,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:17,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:18,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:18,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:18,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:19,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:20,235 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:20,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:21,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:22,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:23,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:24,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:25,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:26,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:26,236 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:27,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:28,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:29,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:30,237 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:31,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:32,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:32,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:32,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:19:33,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:33,118 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:33,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:34,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:35,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:36,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:37,238 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:37,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:38,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:39,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:40,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:41,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:42,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:43,132 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:43,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:44,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:45,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:46,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:47,239 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:48,118 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:19:48,119 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:19:48,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:48,249 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:49,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:50,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:51,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:52,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:53,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:54,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:54,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:55,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:56,240 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:57,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:58,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:19:59,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:19:59,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:00,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:01,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:02,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:02,780 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:20:03,119 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:20:03,119 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:20:03,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:04,202 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:04,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:05,241 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:06,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:07,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:08,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:09,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:10,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:10,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:11,242 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:12,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:13,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:14,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:15,133 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:15,243 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:15,329 DEBUG SenderThread:2033938 [sender.py:send():379] send: exit +2025-01-04 22:20:15,329 INFO SenderThread:2033938 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_exit():588] handling runtime: 492 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_exit():594] send defer +2025-01-04 22:20:15,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,330 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:20:15,330 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:20:15,330 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:20:15,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:20:15,331 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,331 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:20:15,331 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:20:15,331 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:20:15,331 INFO HandlerThread:2033938 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:20:15,331 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:20:15,331 DEBUG SystemMonitor:2033938 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:20:15,332 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:20:15,332 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:20:15,383 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:20:15,384 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:20:15,384 INFO HandlerThread:2033938 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:20:15,384 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,384 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:20:15,384 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:20:15,384 DEBUG SenderThread:2033938 [sender.py:send():379] send: stats +2025-01-04 22:20:15,385 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,385 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:20:15,385 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:15,385 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:20:15,385 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:20:15,385 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:15,385 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:20:15,835 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:17,836 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:21,644 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:20:21,645 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,645 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:20:21,645 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:20:21,645 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:21,646 DEBUG SenderThread:2033938 [sender.py:send():379] send: history +2025-01-04 22:20:21,646 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:20:21,646 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:21,646 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,646 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:20:21,647 DEBUG SenderThread:2033938 [sender.py:send():379] send: summary +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:20:21,647 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:20:21,647 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,648 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:20:21,648 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,648 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:20:21,648 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:20:21,648 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,649 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:20:21,649 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,649 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:20:21,753 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:20:21,753 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,753 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:20:21,753 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,753 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:20:21,754 INFO SenderThread:2033938 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:20:21,756 INFO SenderThread:2033938 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:20:21,758 INFO SenderThread:2033938 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:20:21,769 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:20:21,769 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:21,769 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:20:21,769 DEBUG SenderThread:2033938 [sender.py:send():379] send: artifact +2025-01-04 22:20:21,838 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:21,838 INFO Thread-12 :2033938 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEwMTI1NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 22:20:21,938 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:20:21,938 INFO SenderThread:2033938 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:20:22,330 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml config.yaml +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json wandb-summary.json +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt requirements.txt +2025-01-04 22:20:22,839 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:20:22,840 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log output.log +2025-01-04 22:20:22,843 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/diff.patch diff.patch +2025-01-04 22:20:22,845 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 media/videos/eval/2_eval_2_10e1974516cae1f65fc3.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 media/videos/eval/3_eval_3_129e9d1f1ab8c01c94d0.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 media/videos/eval/4_eval_4_4792f0829cf66c6b9d8a.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 media/videos/eval/0_eval_0_92869d54d5ca1018f4a9.mp4 +2025-01-04 22:20:22,848 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 media/videos/eval/1_eval_1_363a77cf1a9e04df3395.mp4 +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json media/table/eval/total_success_rate_ci_6_f6c18c2ba742daffddcb.table.json +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:20:22,849 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:22,849 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:22,849 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:20:22,849 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:20:22,849 INFO SenderThread:2033938 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:20:23,011 INFO wandb-upload_0:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/wandb-summary.json +2025-01-04 22:20:23,030 INFO wandb-upload_1:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/config.yaml +2025-01-04 22:20:23,083 INFO wandb-upload_3:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/requirements.txt +2025-01-04 22:20:23,083 INFO wandb-upload_2:2033938 [upload_job.py:push():130] Uploaded file /tmp/tmpv289gai6wandb/qt0vmmye-events.out.tfevents.1736046723.leonmkim-ROG-Strix-G15CS-G15CS.2033763.0 +2025-01-04 22:20:23,104 INFO wandb-upload_4:2033938 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/files/output.log +2025-01-04 22:20:23,304 INFO Thread-11 (_thread_body):2033938 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:20:23,304 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,304 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:20:23,305 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,305 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:20:23,305 INFO SenderThread:2033938 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:20:23,306 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,306 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:20:23,306 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:20:23,306 INFO SenderThread:2033938 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:20:23,331 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,578 INFO SenderThread:2033938 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:20:23,578 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:20:23,578 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,578 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,578 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send():379] send: final +2025-01-04 22:20:23,579 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send():379] send: footer +2025-01-04 22:20:23,579 INFO HandlerThread:2033938 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:20:23,579 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: defer +2025-01-04 22:20:23,579 INFO SenderThread:2033938 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:20:23,580 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:20:23,581 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,581 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:20:23,581 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:20:23,646 DEBUG SenderThread:2033938 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:20:23,646 INFO MainThread:2033938 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:20:23,646 INFO MainThread:2033938 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:20:23,647 INFO MainThread:2033938 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:20:23,647 DEBUG HandlerThread:2033938 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:20:23,647 INFO HandlerThread:2033938 [handler.py:finish():882] shutting down handler +2025-01-04 22:20:24,581 INFO WriterThread:2033938 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [sender.py:finish():1615] shutting down sender +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:20:24,646 INFO SenderThread:2033938 [file_pusher.py:join():175] waiting for file pusher diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug.log b/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..79efe5bbe7d02b45fb6529b726a72a16d0dddc5d --- /dev/null +++ b/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Configure stats pid to 2033763 +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/settings +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug.log +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/wandb/run-20250104_221202-br14p9rb/logs/debug-internal.log +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb'} +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():619] starting backend +2025-01-04 22:12:02,376 INFO MainThread:2033763 [wandb_init.py:init():623] setting up manager +2025-01-04 22:12:02,380 INFO MainThread:2033763 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:12:02,381 INFO MainThread:2033763 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:12:02,393 INFO MainThread:2033763 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:12:02,402 INFO MainThread:2033763 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:12:02,677 INFO MainThread:2033763 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:12:02,717 INFO MainThread:2033763 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:12:02,717 INFO MainThread:2033763 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:12:03,102 INFO MainThread:2033763 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:12:03,103 INFO MainThread:2033763 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:12:03,104 INFO MainThread:2033763 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:12:03,104 INFO MainThread:2033763 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/br14p9rb/tb, True +2025-01-04 22:12:08,062 INFO MainThread:2033763 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:20:24,647 WARNING MsgRouterThr:2033763 [router.py:message_loop():77] message_loop has been closed diff --git a/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb b/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fa7f529c39e06386c42dee4083afdb3ee29720e4 Binary files /dev/null and b/br14p9rb/wandb/run-20250104_221202-br14p9rb/run-br14p9rb.wandb differ diff --git a/kdczwpup/episode_rosbags/episode_4_2025-01-03-20-56-24.bag b/kdczwpup/episode_rosbags/episode_4_2025-01-03-20-56-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..3b2cb576a798552d8cab6d1f112ff926db09c711 --- /dev/null +++ b/kdczwpup/episode_rosbags/episode_4_2025-01-03-20-56-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f94fea8c11667006cf60b39d667722ecf6c82c1e66a94e3529f7d415e59adea7 +size 2603563254 diff --git a/mn846g41/.hydra/config.yaml b/mn846g41/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/mn846g41/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/mn846g41/.hydra/hydra.yaml b/mn846g41/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e7874738a9860a60af608d17ab57ff2fc5e76e3 --- /dev/null +++ b/mn846g41/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/mn846g41 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/mn846g41/.hydra/overrides.yaml b/mn846g41/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/mn846g41/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/mn846g41/eval_policy.log b/mn846g41/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..1e6c945ffd60c0a075e1839bfdfdee50ab9fd4bf --- /dev/null +++ b/mn846g41/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 20:26:21,313][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:26:21,316][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 20:26:24,845][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/nbjbj4mc/episode_rosbags/episode_0_2025-01-03-21-06-34.bag b/nbjbj4mc/episode_rosbags/episode_0_2025-01-03-21-06-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..d8c6370d271ee81aa59991c103cff8f476b619bb --- /dev/null +++ b/nbjbj4mc/episode_rosbags/episode_0_2025-01-03-21-06-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82473343a27a75c1577a28f0047930c6b76822a5be1725bf7e78579f02f6cea +size 2572035948 diff --git a/nbjbj4mc/episode_rosbags/episode_2_2025-01-03-21-08-18.bag b/nbjbj4mc/episode_rosbags/episode_2_2025-01-03-21-08-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..2d61223083e15a1f8dfa9240f34d55406649c7b4 --- /dev/null +++ b/nbjbj4mc/episode_rosbags/episode_2_2025-01-03-21-08-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03aae3c3cdad1b5e93dc91f6eb07b82fe63d1a105556596059860123bd1cea6d +size 2620600590 diff --git a/nbjbj4mc/episode_rosbags/episode_3_2025-01-03-21-09-09.bag b/nbjbj4mc/episode_rosbags/episode_3_2025-01-03-21-09-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..135603d661b90b5ab99f1b86c8d9b0d8cdd70f57 --- /dev/null +++ b/nbjbj4mc/episode_rosbags/episode_3_2025-01-03-21-09-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9d6d2894edd6a8f7b877865107b376c617cc625611ec40420923e7f9488b334 +size 2603886913 diff --git a/nbjbj4mc/episode_rosbags/episode_4_2025-01-03-21-10-31.bag b/nbjbj4mc/episode_rosbags/episode_4_2025-01-03-21-10-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..3dd640b8e06a60ca55bd433d78c180e79a420aa3 --- /dev/null +++ b/nbjbj4mc/episode_rosbags/episode_4_2025-01-03-21-10-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:294ea30ca09baaaa8f0155c511b3d1b956b5b29681815afbdf0725f9ec5c0245 +size 2610985364 diff --git a/nibxxtai/episode_rosbags/episode_0_2025-01-03-20-46-55.bag b/nibxxtai/episode_rosbags/episode_0_2025-01-03-20-46-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..8ef141708caf8655504784a77da637bb5148e933 --- /dev/null +++ b/nibxxtai/episode_rosbags/episode_0_2025-01-03-20-46-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d52673f54ee6712b1313e45e24aa99b08bfc3cd60000764bf04a15220b32d81f +size 2621123549 diff --git a/nibxxtai/episode_rosbags/episode_1_2025-01-03-20-47-48.bag b/nibxxtai/episode_rosbags/episode_1_2025-01-03-20-47-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..b8aada23d68edc64a0f0daf8cab2d0a45b6d2db7 --- /dev/null +++ b/nibxxtai/episode_rosbags/episode_1_2025-01-03-20-47-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa2c9b5195684ee52ddf7c3c376b0154c56a1e62241a9ab93163bfad0834c183 +size 2601275775 diff --git a/nibxxtai/episode_rosbags/episode_2_2025-01-03-20-48-40.bag b/nibxxtai/episode_rosbags/episode_2_2025-01-03-20-48-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..991cb58d7fce5e2678d3314ba983ca4fab176422 --- /dev/null +++ b/nibxxtai/episode_rosbags/episode_2_2025-01-03-20-48-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2f7ea0b8b588355a7b1ed5e9c07f6cbf1ed265f62335dbc9f4bda836e492d38 +size 2619785050 diff --git a/nibxxtai/episode_rosbags/episode_3_2025-01-03-20-50-04.bag b/nibxxtai/episode_rosbags/episode_3_2025-01-03-20-50-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..8b025dc8671f2003b7d8b6b8994f6a1798e37428 --- /dev/null +++ b/nibxxtai/episode_rosbags/episode_3_2025-01-03-20-50-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:361020e5e664783941bc28e2b290706e14713a861f3fa6ec6d45223669345fed +size 2600981900 diff --git a/nibxxtai/episode_rosbags/episode_4_2025-01-03-20-51-04.bag b/nibxxtai/episode_rosbags/episode_4_2025-01-03-20-51-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..1431c592cac2f4ed9a57564fa48ab3e613a01c7d --- /dev/null +++ b/nibxxtai/episode_rosbags/episode_4_2025-01-03-20-51-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f88991033743225753cabd9d2b1853795c0279638d172ed524e2618fdf9c31 +size 2608873145 diff --git a/p7zqzp4h/episode_rosbags/episode_0_2025-01-03-23-05-20.bag b/p7zqzp4h/episode_rosbags/episode_0_2025-01-03-23-05-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..a605034fc9a9e176e1eabe57806674d6550c26ec --- /dev/null +++ b/p7zqzp4h/episode_rosbags/episode_0_2025-01-03-23-05-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8edd3182602f33ab90d6becafc332f50150880ce6059386ebb4b74a15c0230d3 +size 2622599052 diff --git a/p7zqzp4h/episode_rosbags/episode_1_2025-01-03-23-06-10.bag b/p7zqzp4h/episode_rosbags/episode_1_2025-01-03-23-06-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..9d337483ef896848058e0e9b6b2ab3c29514f5f9 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/episode_1_2025-01-03-23-06-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b2897a87bb8fbf301fecfd68bfa00b05859f27535fcc4fafde564f637ac896 +size 2609665480 diff --git a/p7zqzp4h/episode_rosbags/episode_2_2025-01-03-23-06-58.bag b/p7zqzp4h/episode_rosbags/episode_2_2025-01-03-23-06-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..509122953770baf70b9edc6cc758f1b88f19e153 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/episode_2_2025-01-03-23-06-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8369e69f62c7fd5ffcf61691f7b29772e2a6586701d86df5600420f08713c11b +size 2610313958 diff --git a/p7zqzp4h/episode_rosbags/episode_3_2025-01-03-23-07-55.bag b/p7zqzp4h/episode_rosbags/episode_3_2025-01-03-23-07-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..59b7554d4ebadf0fadaaaf610ee5774dcabf2523 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/episode_3_2025-01-03-23-07-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07f7b9b7fea3de93a8ada6b448635c9e412bedd7bb711d22067e54bfde947ef +size 2609021904 diff --git a/p7zqzp4h/episode_rosbags/episode_4_2025-01-03-23-08-49.bag b/p7zqzp4h/episode_rosbags/episode_4_2025-01-03-23-08-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..4fd618f40f03028cb35f90cd153bd95ff2c0e265 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/episode_4_2025-01-03-23-08-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:306b84ea89a5f2ead8f52db49944a90995c6fee66867bd9fc2ba04bbfd10a23d +size 2603927537