diff --git a/.gitattributes b/.gitattributes index 4b8c197e77529c5ac8f04bef789c9f92264b3f34..d031958a92bd74db6647817b86f3c8c6d26b098d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -105,3 +105,48 @@ r2rih1d8/episode_rosbags/episode_4_2025-01-07-16-50-24.bag filter=lfs diff=lfs m 5ux8hvp6/episode_rosbags/episode_0_2025-01-07-20-30-50.bag filter=lfs diff=lfs merge=lfs -text 5ux8hvp6/episode_rosbags/episode_4_2025-01-07-20-33-11.bag filter=lfs diff=lfs merge=lfs -text 5ux8hvp6/episode_rosbags/episode_2_2025-01-07-20-31-57.bag filter=lfs diff=lfs merge=lfs -text +h2bqbukm/episode_rosbags/episode_1_2025-01-06-17-51-12.bag filter=lfs diff=lfs merge=lfs -text +h2bqbukm/episode_rosbags/episode_0_2025-01-06-17-50-15.bag filter=lfs diff=lfs merge=lfs -text +5ux8hvp6/episode_rosbags/episode_3_2025-01-07-20-32-29.bag filter=lfs diff=lfs merge=lfs -text +hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 filter=lfs diff=lfs merge=lfs -text +5ux8hvp6/episode_rosbags/episode_1_2025-01-07-20-31-22.bag filter=lfs diff=lfs merge=lfs -text +hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hqgv52wz/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8m5srpvj/episode_rosbags/episode_3_2025-01-06-20-24-01.bag filter=lfs diff=lfs merge=lfs -text +8m5srpvj/episode_rosbags/episode_2_2025-01-06-20-23-12.bag filter=lfs diff=lfs merge=lfs -text +hqgv52wz/episode_rosbags/episode_4_2025-01-07-20-29-14.bag filter=lfs diff=lfs merge=lfs -text +8m5srpvj/episode_rosbags/episode_4_2025-01-06-20-24-51.bag filter=lfs diff=lfs merge=lfs -text +hqgv52wz/episode_rosbags/episode_1_2025-01-07-20-27-28.bag filter=lfs diff=lfs merge=lfs -text +hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/episode_rosbags/episode_0_2025-01-06-22-28-49.bag filter=lfs diff=lfs merge=lfs -text +hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hxim6a4m/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/episode_rosbags/episode_1_2025-01-06-22-29-43.bag filter=lfs diff=lfs merge=lfs -text +hcdwvdao/episode_rosbags/episode_0_2025-01-07-18-06-41.bag filter=lfs diff=lfs merge=lfs -text +guo9e7cv/episode_rosbags/episode_2_2025-01-06-22-30-32.bag filter=lfs diff=lfs merge=lfs -text +1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1jous172/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +guo9e7cv/episode_rosbags/episode_3_2025-01-06-22-31-31.bag filter=lfs diff=lfs merge=lfs -text +8m5srpvj/episode_rosbags/episode_1_2025-01-06-20-22-24.bag filter=lfs diff=lfs merge=lfs -text diff --git a/1jous172/.hydra/config.yaml b/1jous172/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/1jous172/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/1jous172/.hydra/hydra.yaml b/1jous172/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad2e66c798e46e3eda7cd0d8cf4aa3e8a3778cf --- /dev/null +++ b/1jous172/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/1jous172/.hydra/overrides.yaml b/1jous172/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/1jous172/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/1jous172/episode_rosbags/aligned_depth_to_color_K.npy b/1jous172/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1jous172/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1jous172/episode_rosbags/cam_tf_world.npy b/1jous172/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/1jous172/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/1jous172/episode_rosbags/color_K.npy b/1jous172/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1jous172/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1jous172/episode_rosbags/depth_K.npy b/1jous172/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/1jous172/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/1jous172/eval_robot.log b/1jous172/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..9da72b76dee35badf633e69a74d793159ab70192 --- /dev/null +++ b/1jous172/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:39:00,126][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:39:00,129][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/1jous172/eval_video/0_eval.mp4 b/1jous172/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..00be2b11fb934224108d15fd965aa21981c5a500 --- /dev/null +++ b/1jous172/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f9e236adbaeed571af328ec6866b6fffe5499078fe78ffe5946911e9a3c5149 +size 1467200 diff --git a/1jous172/eval_video/1_eval.mp4 b/1jous172/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24beb7f63e5bde3704691378ff5bb57330f707ee --- /dev/null +++ b/1jous172/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04037a9f5a7dfcbf716d536a0d85ff2a0784a07441b2ae19d387585202126198 +size 1444840 diff --git a/1jous172/eval_video/2_eval.mp4 b/1jous172/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88a9de799fa0eb4193ab58ac4124d3df3bfe44b1 --- /dev/null +++ b/1jous172/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c482b456ff56ddc8a1e1dda040b1f65899103d79811a3c1df131c6925197180a +size 1433597 diff --git a/1jous172/eval_video/3_eval.mp4 b/1jous172/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d60077aad277f3ef51f89a255cfd940ad6f76406 --- /dev/null +++ b/1jous172/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77698069b89e74b810139ba27c5e0353a124c9f3c5d93a9f7a6c189960f045ce +size 1436091 diff --git a/1jous172/eval_video/4_eval.mp4 b/1jous172/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7af8dc5082d6d415b259da6479cb4ea3332e0540 --- /dev/null +++ b/1jous172/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:254f22dd2caf14ce117c469e5dfa0992b6bcfa5fb1b8872538465783339d0a53 +size 1427933 diff --git a/1jous172/tb/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 b/1jous172/tb/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 new file mode 100644 index 0000000000000000000000000000000000000000..8972ccbc82d8fc1341d266ceaf49d36edbb515a8 --- /dev/null +++ b/1jous172/tb/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fec104fd5a47f5493e2fc0e80d543c5199007a2a2d3417b4ae24214d221d0456 +size 6539 diff --git a/1jous172/wandb/debug-internal.log b/1jous172/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..633fbc203a5b05e6c006b6291b9b48ce77df1806 --- /dev/null +++ b/1jous172/wandb/debug-internal.log @@ -0,0 +1,1238 @@ +2025-01-06 17:39:06,228 INFO StreamThr :2093159 [internal.py:wandb_internal():85] W&B internal server running at pid: 2093159, started at: 2025-01-06 17:39:06.228202 +2025-01-06 17:39:06,229 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:39:06,241 INFO WriterThread:2093159 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb +2025-01-06 17:39:06,241 DEBUG SenderThread:2093159 [sender.py:send():379] send: header +2025-01-06 17:39:06,252 DEBUG SenderThread:2093159 [sender.py:send():379] send: run +2025-01-06 17:39:06,511 INFO SenderThread:2093159 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files +2025-01-06 17:39:06,511 INFO SenderThread:2093159 [sender.py:_start_run_threads():1188] run started: 1jous172 with start time 1736203146.23001 +2025-01-06 17:39:06,532 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:39:06,532 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:39:06,602 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:39:06,606 DEBUG HandlerThread:2093159 [system_info.py:__init__():26] System info init +2025-01-06 17:39:06,607 DEBUG HandlerThread:2093159 [system_info.py:__init__():41] System info init done +2025-01-06 17:39:06,607 INFO HandlerThread:2093159 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:39:06,607 INFO SystemMonitor:2093159 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:39:06,607 INFO HandlerThread:2093159 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:39:06,608 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:39:06,608 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:39:06,609 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:39:06,609 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:39:06,610 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:39:06,620 DEBUG HandlerThread:2093159 [system_info.py:probe():152] Probing system +2025-01-06 17:39:06,622 DEBUG HandlerThread:2093159 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:probe():200] Probing system done +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:39:06.621054', 'startedAt': '2025-01-06T22:39:06.220965', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.93825, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.946, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 353.0933418273926}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:39:06,627 INFO HandlerThread:2093159 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:39:06,627 INFO HandlerThread:2093159 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:_save_code():44] Saving code +2025-01-06 17:39:06,629 DEBUG HandlerThread:2093159 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:39:06,629 DEBUG HandlerThread:2093159 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:39:06,730 DEBUG HandlerThread:2093159 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:39:06,730 INFO HandlerThread:2093159 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:39:06,733 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:39:06,733 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:39:06,733 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:39:06,734 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:39:06,969 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:39:06,969 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:39:06,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:06,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:06,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:06,972 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/40nlmnbq-wandb-metadata.json +2025-01-06 17:39:06,972 INFO HandlerThread:2093159 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/tb" + save: true +} +control { + relay_id: "469e8a1cb67b419d92c8e71f617843fd" +} +_info { + stream_id: "1jous172" +} + +2025-01-06 17:39:06,973 DEBUG HandlerThread:2093159 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:39:07,025 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/il8kn86k-code/FISH/eval_robot.py +2025-01-06 17:39:07,033 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/tz3pb0wm-diff.patch +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: telemetry +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: tbrecord +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:39:07,042 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 with policy live +2025-01-06 17:39:07,226 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/gdyxm7b1-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code +2025-01-06 17:39:07,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:08,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:09,512 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:09,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:10,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: config +2025-01-06 17:39:10,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:11,627 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:11,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:12,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:13,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:13,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:14,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:15,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:15,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:16,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:16,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:17,515 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:17,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:18,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:19,515 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:19,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:20,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:21,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:21,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:22,055 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:22,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:23,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:24,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:25,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:26,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:27,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:27,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:28,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:29,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:30,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:31,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:32,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:32,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:33,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:34,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:35,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:36,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:36,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:37,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:38,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:38,520 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:39:38,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:39,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:40,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:41,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:42,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:43,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:43,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:44,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:45,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:46,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:47,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:48,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:49,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:49,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:50,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:51,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:51,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:52,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:53,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:54,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:55,055 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:55,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:56,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:57,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:58,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:59,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:00,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:00,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:01,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:02,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:03,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:04,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:05,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:05,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:06,611 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:40:06,611 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:40:06,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:06,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:07,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:08,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:09,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:10,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:11,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:11,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:12,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:13,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:14,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:15,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:16,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:16,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:17,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:18,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:19,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:20,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:21,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:21,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:22,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:22,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:23,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:24,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:25,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:26,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:27,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:27,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:28,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:29,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:30,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:31,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:32,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:32,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:33,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:34,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:35,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:36,612 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:40:36,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:36,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:37,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:38,049 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:38,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:39,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:40,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:41,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:42,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:43,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:43,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:44,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:45,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:46,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:47,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:48,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:48,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:49,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:50,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:51,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:52,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:53,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:54,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:54,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:55,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:56,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:57,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:58,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:59,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:59,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:00,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:01,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:02,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:03,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:04,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:04,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:05,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:06,612 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:41:06,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:06,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:07,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:08,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:09,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:10,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:10,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:11,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:12,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:13,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:14,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:15,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:15,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:16,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:17,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:18,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:19,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:20,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:21,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:21,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:22,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:22,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:23,989 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:24,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:25,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:26,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:27,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:27,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:28,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:29,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:30,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:31,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:32,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:33,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:33,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:34,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:35,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:36,614 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:41:36,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:36,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:37,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:38,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:39,110 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:39,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:40,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:41,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:42,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:43,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:44,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:44,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:45,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:46,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:47,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:48,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:49,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:50,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:50,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:51,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:52,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:53,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:54,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:55,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:56,051 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:56,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:57,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:58,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:59,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:00,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:01,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:01,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:02,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:03,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:04,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:05,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:06,614 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:42:06,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:06,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:07,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:07,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:08,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:09,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:10,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:11,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:12,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:12,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:13,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:14,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:15,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:16,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:17,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:17,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:18,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:19,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:20,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:21,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:21,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:22,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:23,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:23,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:24,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:25,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:26,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:27,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:28,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:28,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:29,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:30,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:31,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:32,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:33,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:33,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:34,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:35,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:36,615 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:42:36,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:36,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:37,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:38,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:39,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:39,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:40,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:41,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:42,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:43,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:44,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:44,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:45,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:47,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:48,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:49,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:49,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:50,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:51,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:51,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:53,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:54,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:55,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:55,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:56,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:57,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:58,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:59,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:00,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:00,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:01,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:02,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:03,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:04,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:05,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:05,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:06,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:06,616 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:43:06,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:06,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:08,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:09,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:10,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:11,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:11,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:12,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:13,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:14,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:15,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:16,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:16,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:17,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:18,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:19,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:20,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:21,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:21,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:22,107 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:23,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:24,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:25,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:26,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:27,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:27,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:28,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:29,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:30,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:31,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:32,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:32,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:33,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:34,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:35,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:36,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:36,617 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:43:36,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:36,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:38,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:38,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:39,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:40,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:41,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:42,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:43,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:43,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:44,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:45,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:46,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:47,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:48,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:49,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:49,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:50,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:51,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:51,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:51,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:53,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:54,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:55,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:55,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:56,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:57,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:58,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:59,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:00,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:00,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:01,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:02,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:03,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:04,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:05,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:06,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:06,617 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:44:06,618 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:06,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:06,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:07,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:08,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:09,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:10,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:11,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:11,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:12,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:13,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:14,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:15,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:16,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:16,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:17,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:18,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:19,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:20,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:21,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:21,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:22,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:22,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:23,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:24,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:25,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:25,605 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:26,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:27,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:27,606 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:27,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:28,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:29,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:29,607 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:30,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:31,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:32,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:32,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:33,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:34,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:35,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:36,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:36,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:44:36,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:36,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:37,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:38,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:38,577 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:39,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:39,609 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:40,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:41,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:42,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:43,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:43,611 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:43,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:44,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:45,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:46,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:47,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:48,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:49,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:49,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:50,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:51,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:51,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:51,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:52,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:53,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:54,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:55,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:55,144 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:56,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:57,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:58,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:59,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:00,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:00,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:01,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:02,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:03,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:04,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:04,564 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:45:04,564 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:45:04,564 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 with policy now +2025-01-06 17:45:04,565 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:45:04,565 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:45:04,566 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos +2025-01-06 17:45:04,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media +2025-01-06 17:45:04,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:45:04,865 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/sqafg0jm-media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:05,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:05,151 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/l2dx4xh3-media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:05,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:06,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:06,579 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:06,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:45:06,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:06,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:07,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:08,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:09,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:09,618 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:10,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:11,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:11,619 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:11,937 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:12,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:13,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:13,619 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:14,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:15,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:16,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:16,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:17,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:18,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:19,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:20,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:21,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:21,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:22,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:22,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:23,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:24,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:25,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:26,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:26,624 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:45:26,870 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/tp0klld9-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:45:27,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:27,417 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:27,624 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:28,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:29,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:29,625 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:30,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:31,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:31,625 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:32,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:32,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:33,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:34,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:35,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:36,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:36,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:45:36,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:36,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:37,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:37,627 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:38,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:38,054 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:39,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:40,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:41,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:41,629 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:42,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:43,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:43,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:44,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:45,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:46,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:47,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:48,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:49,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:49,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:50,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:51,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:51,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:51,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:52,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:53,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:54,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:55,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:55,103 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:56,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:57,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:58,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:59,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:00,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:00,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:01,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:02,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:03,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:04,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:05,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:06,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:06,167 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:06,218 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:46:06,218 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 with policy now +2025-01-06 17:46:06,220 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:46:06,221 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:46:06,221 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:46:06,240 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:46:06,485 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/6h1a6y54-media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:46:06,920 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/j3pwg2pg-media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:06,977 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:07,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:07,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:08,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:09,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:10,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:11,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:11,638 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:11,889 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:12,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:13,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:13,639 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:14,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:15,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:16,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:16,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:17,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:18,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:19,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:20,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:21,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:21,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:21,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:22,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:22,124 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:23,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:24,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:25,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:26,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:27,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:27,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:28,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:29,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:30,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:31,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:32,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:33,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:33,649 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:46:33,651 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:33,790 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:46:33,790 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:46:33,791 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:33,791 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:46:33,924 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/5027jja0-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:46:34,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:34,649 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:46:35,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:35,650 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:36,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:36,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:46:36,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:36,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:37,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:37,650 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:38,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:39,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:39,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:40,034 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:41,034 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:42,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:43,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:44,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:44,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:45,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:46,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:47,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:48,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:49,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:50,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:50,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:51,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:51,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:51,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:52,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:53,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:54,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:55,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:55,656 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:56,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:56,854 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:57,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:58,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:59,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:59,658 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:00,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:01,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:01,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:02,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:03,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:04,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:05,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:06,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:06,620 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:47:06,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:06,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:07,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:07,098 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:08,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:09,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:10,039 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:11,039 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:12,042 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:12,179 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:12,515 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:47:12,515 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 with policy now +2025-01-06 17:47:12,515 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:47:12,516 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:47:12,516 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:47:12,525 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:47:12,791 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/st0fd33d-media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:13,042 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:13,059 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/y6u3pzub-media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:13,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:14,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:15,045 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:16,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:17,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:17,664 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:17,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:18,046 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:19,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:19,664 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:20,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:21,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:21,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:21,988 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:22,045 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:23,046 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:23,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:23,615 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:47:23,615 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:47:23,615 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:24,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:25,048 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:25,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:26,051 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:27,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:27,667 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:28,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:28,117 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:29,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:29,668 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:30,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:31,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:32,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:33,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:33,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:34,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:35,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:36,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:36,621 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:47:36,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:36,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:37,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:38,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:39,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:39,138 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:40,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:41,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:41,672 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:42,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:43,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:44,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:44,185 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:45,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:45,674 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:46,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:47,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:47,675 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:48,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:49,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:49,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:50,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:51,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:51,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:51,977 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:52,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:53,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:54,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:55,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:55,117 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:56,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:57,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:58,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:59,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:00,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:00,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:01,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:02,070 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:02,815 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:48:02,815 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 with policy now +2025-01-06 17:48:02,819 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:48:02,844 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:48:02,844 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:48:02,869 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:48:03,064 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:03,076 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/y7qlbzk4-media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:48:03,976 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/5jr7o643-media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:04,076 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:05,077 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:06,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:06,471 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:06,622 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:48:06,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:06,983 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:07,079 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:07,681 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:08,081 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:09,080 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:10,078 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:11,079 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:11,683 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:12,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:12,083 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:13,080 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:14,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:15,081 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:16,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:17,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:17,082 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:18,083 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:19,086 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:20,086 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:21,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:21,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:48:21,692 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:48:21,692 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:48:21,693 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:48:21,965 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/vxbri0xf-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:48:21,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:21,985 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:22,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:22,140 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:22,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:48:23,096 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:23,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:24,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:25,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:25,687 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:26,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:27,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:27,688 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:28,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:28,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:29,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:30,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:31,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:32,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:33,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:33,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:34,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:35,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:36,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:36,622 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:48:36,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:36,978 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:37,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:38,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:39,077 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:39,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:40,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:41,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:42,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:43,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:44,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:45,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:45,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:46,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:47,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:48,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:49,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:50,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:50,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:51,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:51,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:51,979 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:52,095 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:53,095 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:54,096 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:55,048 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:55,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:56,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:57,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:58,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:59,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:59,699 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:00,102 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:00,567 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:01,120 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:01,177 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:01,177 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 with policy now +2025-01-06 17:49:01,191 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:01,191 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:01,191 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:01,212 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:01,441 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/uhz293fi-media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:01,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:49:01,992 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/nkocmcjf-media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:02,109 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:03,115 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:03,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:04,116 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:05,103 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:05,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:05,897 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:06,115 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:06,623 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:49:06,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:49:06,984 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:49:07,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:08,111 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:09,111 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:09,578 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:49:09,578 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media +2025-01-06 17:49:09,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table +2025-01-06 17:49:09,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval +2025-01-06 17:49:09,787 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:09,890 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/mc9dnymz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:10,042 INFO wandb-upload_2:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptzqc3sdw +2025-01-06 17:49:10,104 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:10,137 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:49:10,177 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:10,480 INFO SenderThread:2093159 [sender.py:send_request_log_artifact():1525] logged artifact run-1jous172-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTE2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxNDc1Nw==', 'latestArtifact': None}} +2025-01-06 17:49:10,481 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:10,481 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 17:49:10,481 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:10,482 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,482 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,482 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:10,483 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,486 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send():379] send: telemetry +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,487 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,487 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,488 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,488 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,488 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,488 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,489 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,489 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,490 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,490 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,490 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,490 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,491 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,491 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,491 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,491 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,492 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,492 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:49:10,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval +2025-01-06 17:49:10,728 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/4kkpa2oc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:10,854 INFO wandb-upload_2:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmrxdjnzp +2025-01-06 17:49:10,930 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/pnjp4gzq-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:11,104 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:11,247 INFO SenderThread:2093159 [sender.py:send_request_log_artifact():1525] logged artifact run-1jous172-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTE5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxNDc2MQ==', 'latestArtifact': None}} +2025-01-06 17:49:11,247 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:11,247 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-06 17:49:11,248 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:11,248 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:11,248 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:11,248 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:11,497 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/wyeihir3-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:11,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:11,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:12,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:13,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:14,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:15,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:16,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:16,578 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:16,621 DEBUG SenderThread:2093159 [sender.py:send():379] send: exit +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():588] handling runtime: 610 +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():594] send defer +2025-01-06 17:49:16,622 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,622 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:49:16,622 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:49:16,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:49:16,623 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:49:16,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:49:16,623 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:49:16,624 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:49:16,624 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:49:16,624 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:49:16,694 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:49:16,694 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:49:16,695 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:49:16,695 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,695 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:49:16,695 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:49:16,695 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,695 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:49:16,695 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:49:16,696 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,696 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:49:16,696 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:49:16,696 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,696 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:49:16,706 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:17,706 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:49:22,759 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,759 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:49:22,759 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:22,760 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,760 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:22,760 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:49:22,761 DEBUG SenderThread:2093159 [sender.py:send():379] send: summary +2025-01-06 17:49:22,764 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:22,764 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:49:22,765 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,765 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:49:22,765 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:49:22,768 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:22,906 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:49:22,906 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,906 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:49:22,906 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,906 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:49:23,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:23,709 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:23,709 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:49:25,145 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:49:25,145 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:25,145 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:25,145 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:49:25,146 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:25,146 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:49:25,146 INFO SenderThread:2093159 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:49:25,148 INFO SenderThread:2093159 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:49:25,150 INFO SenderThread:2093159 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:49:25,160 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:49:25,161 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:25,161 DEBUG SenderThread:2093159 [sender.py:send():379] send: artifact +2025-01-06 17:49:25,161 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:49:25,620 INFO wandb-upload_2:2093159 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwscw5le0 +2025-01-06 17:49:25,620 INFO wandb-upload_1:2093159 [upload_job.py:push():85] Skipped uploading /tmp/tmp3ojk991x/wandb-job.json +2025-01-06 17:49:25,624 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:25,710 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:25,726 INFO wandb-upload_0:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnh0s_z63 +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTYxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'versionIndex': 424}}} +2025-01-06 17:49:26,354 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml config.yaml +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json wandb-summary.json +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt requirements.txt +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log output.log +2025-01-06 17:49:26,715 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch diff.patch +2025-01-06 17:49:26,716 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:26,720 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:49:26,720 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:26,722 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:26,722 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:49:26,722 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:26,722 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:49:26,722 INFO SenderThread:2093159 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:49:26,897 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:26,912 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:49:26,946 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt +2025-01-06 17:49:26,974 INFO wandb-upload_4:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:26,995 INFO wandb-upload_3:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/ie0stzsg-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:27,195 INFO Thread-11 (_thread_body):2093159 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:49:27,195 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,195 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:49:27,196 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,196 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:49:27,196 INFO SenderThread:2093159 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:49:27,197 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,197 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:49:27,197 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:49:27,407 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,407 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:49:27,407 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:49:27,407 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,407 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send():379] send: final +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send():379] send: footer +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,408 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:49:27,408 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:27,409 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:49:27,410 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:49:27,411 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:49:27,413 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:27,469 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:49:27,471 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:49:27,471 INFO HandlerThread:2093159 [handler.py:finish():882] shutting down handler +2025-01-06 17:49:28,409 INFO WriterThread:2093159 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [sender.py:finish():1615] shutting down sender +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [file_pusher.py:join():175] waiting for file pusher diff --git a/1jous172/wandb/debug.log b/1jous172/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e3492409c088702263f5689db87190dd05bd27e9 --- /dev/null +++ b/1jous172/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Configure stats pid to 2092961 +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/settings +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/logs/debug.log +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/logs/debug-internal.log +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172'} +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():619] starting backend +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():623] setting up manager +2025-01-06 17:39:06,227 INFO MainThread:2092961 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:39:06,229 INFO MainThread:2092961 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:39:06,240 INFO MainThread:2092961 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:39:06,249 INFO MainThread:2092961 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:39:06,532 INFO MainThread:2092961 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:39:06,596 INFO MainThread:2092961 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:39:06,597 INFO MainThread:2092961 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:39:06,971 INFO MainThread:2092961 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:39:06,972 INFO MainThread:2092961 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/tb, True +2025-01-06 17:39:10,618 INFO MainThread:2092961 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:49:28,471 WARNING MsgRouterThr:2092961 [router.py:message_loop():77] message_loop has been closed diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py b/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml b/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f72460028806074afa158d773db7b75bacecb43b --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736203146 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch b/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..5c450f986014be4c83153c4c0f7669b2ae0a5569 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..f19781e 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json b/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd62798ae2246ea155c51d2c7f88d715ecd78c3 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.43520843983992363, 1, 0.60740816591022, 1, 0.6842965185273651, 1, 0.6614799052153926, 1]]} \ No newline at end of file diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..00be2b11fb934224108d15fd965aa21981c5a500 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f9e236adbaeed571af328ec6866b6fffe5499078fe78ffe5946911e9a3c5149 +size 1467200 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24beb7f63e5bde3704691378ff5bb57330f707ee --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04037a9f5a7dfcbf716d536a0d85ff2a0784a07441b2ae19d387585202126198 +size 1444840 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88a9de799fa0eb4193ab58ac4124d3df3bfe44b1 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c482b456ff56ddc8a1e1dda040b1f65899103d79811a3c1df131c6925197180a +size 1433597 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d60077aad277f3ef51f89a255cfd940ad6f76406 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77698069b89e74b810139ba27c5e0353a124c9f3c5d93a9f7a6c189960f045ce +size 1436091 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7af8dc5082d6d415b259da6479cb4ea3332e0540 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:254f22dd2caf14ce117c469e5dfa0992b6bcfa5fb1b8872538465783339d0a53 +size 1427933 diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/output.log b/1jous172/wandb/run-20250106_173906-1jous172/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b586a29a87323da968b4051c1aa2d0684b65bea4 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/output.log @@ -0,0 +1,273 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736203150.619110]: resetting environment +[INFO] [1736203150.623609]: cleared current plan +[INFO] [1736203150.624615]: moving to home +[INFO] [1736203151.626705]: reached home +[INFO] [1736203151.628045]: reset action history +[INFO] [1736203154.338404]: environment reset +Starting episode 0 +[INFO] [1736203154.345202]: resetting environment +[INFO] [1736203154.353118]: cleared current plan +[INFO] [1736203154.354352]: moving to home +[INFO] [1736203155.356545]: reached home +[INFO] [1736203155.357918]: reset action history +[INFO] [1736203158.068300]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736203464.662712]: resetting environment +[INFO] [1736203464.667566]: cleared current plan +[INFO] [1736203464.668391]: moving to home +[INFO] [1736203465.671040]: reached home +[INFO] [1736203465.672437]: reset action history +[INFO] [1736203468.380109]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/episode_rosbags/episode_0_2025-01-06-17-44-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2092961.log +started roslaunch server http://158.130.50.37:41891/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2093685] +process[print_text-2]: started with pid [2093709] +process[rosbag_record-3]: started with pid [2093710] +started bagging! +For topic gripper_width: timestamp difference is -110784299 for nearest: 1736203477340116732 - target: 1736203477450901031 at idx 121 +For topic gripper_width: timestamp difference is -107298625 for nearest: 1736203481740118014 - target: 1736203481847416639 at idx 178 +[INFO] [1736203503.544140]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736203504.357831]: Stored episode 1. +[INFO] [1736203504.358094]: Saving video... +[INFO] [1736203504.554432]: Video saved! +Starting episode 1 +[INFO] [1736203504.567895]: resetting environment +[INFO] [1736203504.575504]: cleared current plan +[INFO] [1736203504.578755]: moving to home +[INFO] [1736203508.182657]: reached home +[INFO] [1736203508.184089]: reset action history +[INFO] [1736203510.893190]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736203526.410375]: resetting environment +[INFO] [1736203526.415829]: cleared current plan +[INFO] [1736203526.416024]: moving to home +[INFO] [1736203527.417331]: reached home +[INFO] [1736203527.417731]: reset action history +[INFO] [1736203530.125568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/episode_rosbags/episode_1_2025-01-06-17-45-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2092961.log +started roslaunch server http://158.130.50.37:46237/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2093835] +process[print_text-5]: started with pid [2093836] +process[rosbag_record-6]: started with pid [2093860] +started bagging! +For topic gripper_width: timestamp difference is 103337832 for nearest: 1736203536306814023 - target: 1736203536203476191 at idx 73 +For topic gripper_width: timestamp difference is -224928640 for nearest: 1736203539142697312 - target: 1736203539367625952 at idx 107 +[INFO] [1736203565.046741]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736203565.773892]: Stored episode 2. +[INFO] [1736203565.774467]: Saving video... +[INFO] [1736203566.165068]: Video saved! +Starting episode 2 +[INFO] [1736203566.223458]: resetting environment +[INFO] [1736203566.242556]: cleared current plan +[INFO] [1736203566.242842]: moving to home +[INFO] [1736203569.846637]: reached home +[INFO] [1736203569.852560]: reset action history +[INFO] [1736203572.591933]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736203592.951069]: resetting environment +[INFO] [1736203592.959370]: cleared current plan +[INFO] [1736203592.959576]: moving to home +[INFO] [1736203593.961252]: reached home +[INFO] [1736203593.961751]: reset action history +[INFO] [1736203596.673191]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/episode_rosbags/episode_2_2025-01-06-17-46-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2092961.log +started roslaunch server http://158.130.50.37:41683/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2093990] +process[print_text-8]: started with pid [2093991] +process[rosbag_record-9]: started with pid [2093992] +started bagging! +For topic gripper_width: timestamp difference is 102447019 for nearest: 1736203613806780801 - target: 1736203613704333782 at idx 224 +For topic gripper_width: timestamp difference is 102458063 for nearest: 1736203617706808153 - target: 1736203617604350090 at idx 249 +[INFO] [1736203631.456456]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736203632.178770]: Stored episode 3. +[INFO] [1736203632.181919]: Saving video... +[INFO] [1736203632.471110]: Video saved! +Starting episode 3 +[INFO] [1736203632.522521]: resetting environment +[INFO] [1736203632.574780]: cleared current plan +[INFO] [1736203632.575605]: moving to home +[INFO] [1736203636.180811]: reached home +[INFO] [1736203636.186755]: reset action history +[INFO] [1736203638.975089]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736203643.078753]: resetting environment +[INFO] [1736203643.103116]: cleared current plan +[INFO] [1736203643.105714]: moving to home +[INFO] [1736203644.109034]: reached home +[INFO] [1736203644.109821]: reset action history +[INFO] [1736203646.834115]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/episode_rosbags/episode_3_2025-01-06-17-47-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2092961.log +started roslaunch server http://158.130.50.37:33175/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2094130] +process[print_text-11]: started with pid [2094131] +process[rosbag_record-12]: started with pid [2094155] +started bagging! +For topic gripper_width: timestamp difference is 103787042 for nearest: 1736203660806782104 - target: 1736203660702995062 at idx 196 +For topic gripper_width: timestamp difference is 105641538 for nearest: 1736203663140113050 - target: 1736203663034471512 at idx 225 +For topic gripper_width: timestamp difference is -158350646 for nearest: 1736203666040144742 - target: 1736203666198495388 at idx 249 +[INFO] [1736203681.626204]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736203682.416232]: Stored episode 4. +[INFO] [1736203682.425322]: Saving video... +[INFO] [1736203682.751906]: Video saved! +Starting episode 4 +[INFO] [1736203682.832152]: resetting environment +[INFO] [1736203682.863613]: cleared current plan +[INFO] [1736203682.863866]: moving to home +[INFO] [1736203686.469206]: reached home +[INFO] [1736203686.478070]: reset action history +[INFO] [1736203689.280244]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736203701.164654]: resetting environment +[INFO] [1736203701.200216]: cleared current plan +[INFO] [1736203701.204398]: moving to home +[INFO] [1736203702.210950]: reached home +[INFO] [1736203702.213961]: reset action history +[INFO] [1736203704.947378]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/episode_rosbags/episode_4_2025-01-06-17-48-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2092961.log +started roslaunch server http://158.130.50.37:41121/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2094279] +process[print_text-14]: started with pid [2094280] +process[rosbag_record-15]: started with pid [2094281] +started bagging! +For topic gripper_width: timestamp difference is 107566491 for nearest: 1736203706506781712 - target: 1736203706399215221 at idx 24 +For topic gripper_width: timestamp difference is 100411579 for nearest: 1736203739006830141 - target: 1736203738906418562 at idx 249 +[INFO] [1736203739.766288]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736203740.565830]: Stored episode 5. +[INFO] [1736203740.575666]: Saving video... +[INFO] [1736203741.097787]: Video saved! +[WARN] [1736203741.119946]: Plan exhausted +[WARN] [1736203741.157131]: Plan exhausted +[WARN] [1736203741.196885]: Plan exhausted +[INFO] [1736203741.192215]: resetting environment +[INFO] [1736203741.222224]: cleared current plan +[INFO] [1736203741.228599]: moving to home +[INFO] [1736203744.854486]: reached home +[INFO] [1736203744.857725]: reset action history +[INFO] [1736203747.665661]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt b/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json b/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fb7741b4d9dbf49a3ddbe1ee7c74ebee7864413f --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:39:06.621054", + "startedAt": "2025-01-06T22:39:06.220965", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.93825, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.946, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 353.0933418273926 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json b/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..796fab6e253b925284ae56c5c922af1b79c7a7d2 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "1f9e236adbaeed571af328ec6866b6fffe5499078fe78ffe5946911e9a3c5149", "size": 1467200, "path": "media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4"}, "global_step": 665, "_timestamp": 1736203750.0376449, "_runtime": 603.8076348304749, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "04037a9f5a7dfcbf716d536a0d85ff2a0784a07441b2ae19d387585202126198", "size": 1444840, "path": "media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "c482b456ff56ddc8a1e1dda040b1f65899103d79811a3c1df131c6925197180a", "size": 1433597, "path": "media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "77698069b89e74b810139ba27c5e0353a124c9f3c5d93a9f7a6c189960f045ce", "size": 1436091, "path": "media/videos/eval/3_eval_3_77698069b89e74b81013.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "254f22dd2caf14ce117c469e5dfa0992b6bcfa5fb1b8872538465783339d0a53", "size": 1427933, "path": "media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://8mcl8taztcnm6oujksa72ggxk3mxhbq6jwyoex7s2l0p5ny7xu0lxrj1lgec1xgbzibz3upvlmjhge2au3bwf4gipo3lswkd893lkci4rurhia4jlufpd1pew4vuqqtg/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://knlz3rpczrlxj5lvjee24tu633hbbkbwmmn3vg0iijxeh8nt025u9vv5oayi4l1x86i9w99zyhzr8nzwvjxy9ccli9vpf42221ixpdml2ljxke59x0r8eip9ohaivp8s:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.43520843983992363, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.60740816591022, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.6842965185273651, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.6614799052153926, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e32f206babc2825ce21715b938b57354f64231e089822aca554fad29505cf476", "size": 434, "artifact_path": "wandb-client-artifact://enby3pmzaqq5bwby7v1ftk2x82b7j0mo2iwz8ve96x6ra5wy8aeepag17lgvsvicdbenw99c0szkpv7v3cy6lgk1wr16k8zvn4ifmchyxujqlqf5427ydwpsvhiwsobf/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://wboqtvcjnf4qdtw2al141riwbryeqcuw9xbj9n4zttw8imcovfnqiijg0p3sdbld22vegkvr1wd8d9jxj4wkmdp84yw78odtossaumvb7u00so4mzexqx8o46gc31e7p:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 610}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.43520843982696533, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.6074081659317017, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.6842964887619019, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.6614798903465271, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/1jous172/wandb/run-20250106_173906-1jous172/logs/debug-internal.log b/1jous172/wandb/run-20250106_173906-1jous172/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..633fbc203a5b05e6c006b6291b9b48ce77df1806 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/logs/debug-internal.log @@ -0,0 +1,1238 @@ +2025-01-06 17:39:06,228 INFO StreamThr :2093159 [internal.py:wandb_internal():85] W&B internal server running at pid: 2093159, started at: 2025-01-06 17:39:06.228202 +2025-01-06 17:39:06,229 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:39:06,241 INFO WriterThread:2093159 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb +2025-01-06 17:39:06,241 DEBUG SenderThread:2093159 [sender.py:send():379] send: header +2025-01-06 17:39:06,252 DEBUG SenderThread:2093159 [sender.py:send():379] send: run +2025-01-06 17:39:06,511 INFO SenderThread:2093159 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files +2025-01-06 17:39:06,511 INFO SenderThread:2093159 [sender.py:_start_run_threads():1188] run started: 1jous172 with start time 1736203146.23001 +2025-01-06 17:39:06,532 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:39:06,532 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:39:06,602 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:39:06,606 DEBUG HandlerThread:2093159 [system_info.py:__init__():26] System info init +2025-01-06 17:39:06,607 DEBUG HandlerThread:2093159 [system_info.py:__init__():41] System info init done +2025-01-06 17:39:06,607 INFO HandlerThread:2093159 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:39:06,607 INFO SystemMonitor:2093159 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:39:06,607 INFO HandlerThread:2093159 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:39:06,608 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:39:06,608 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:39:06,609 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:39:06,609 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:39:06,610 INFO SystemMonitor:2093159 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:39:06,620 DEBUG HandlerThread:2093159 [system_info.py:probe():152] Probing system +2025-01-06 17:39:06,622 DEBUG HandlerThread:2093159 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:probe():200] Probing system done +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:39:06.621054', 'startedAt': '2025-01-06T22:39:06.220965', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.93825, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.946, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 353.0933418273926}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:39:06,627 INFO HandlerThread:2093159 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:39:06,627 INFO HandlerThread:2093159 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:39:06,627 DEBUG HandlerThread:2093159 [system_info.py:_save_code():44] Saving code +2025-01-06 17:39:06,629 DEBUG HandlerThread:2093159 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:39:06,629 DEBUG HandlerThread:2093159 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:39:06,730 DEBUG HandlerThread:2093159 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:39:06,730 INFO HandlerThread:2093159 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:39:06,733 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:39:06,733 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:39:06,733 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:39:06,734 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:39:06,969 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:39:06,969 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:39:06,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:06,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:06,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:06,972 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/40nlmnbq-wandb-metadata.json +2025-01-06 17:39:06,972 INFO HandlerThread:2093159 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/tb" + save: true +} +control { + relay_id: "469e8a1cb67b419d92c8e71f617843fd" +} +_info { + stream_id: "1jous172" +} + +2025-01-06 17:39:06,973 DEBUG HandlerThread:2093159 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:39:07,025 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/il8kn86k-code/FISH/eval_robot.py +2025-01-06 17:39:07,033 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/tz3pb0wm-diff.patch +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: telemetry +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: tbrecord +2025-01-06 17:39:07,042 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:39:07,042 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 with policy live +2025-01-06 17:39:07,226 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/gdyxm7b1-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:07,513 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH +2025-01-06 17:39:07,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code +2025-01-06 17:39:07,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:08,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:09,512 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:09,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:10,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: config +2025-01-06 17:39:10,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:11,627 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:11,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:12,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:13,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:13,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:14,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:15,514 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:15,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:16,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:16,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:17,515 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:17,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:18,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:19,515 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:39:19,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:20,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:21,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:21,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:22,055 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:22,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:23,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:24,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:25,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:26,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:27,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:27,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:28,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:29,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:30,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:31,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:32,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:32,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:33,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:34,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:35,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:36,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:36,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:37,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:38,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:38,520 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:39:38,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:39,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:40,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:41,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:42,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:43,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:43,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:44,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:45,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:46,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:47,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:48,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:49,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:49,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:50,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:51,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:39:51,970 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:39:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:52,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:53,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:54,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:55,055 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:39:55,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:56,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:57,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:58,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:39:59,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:00,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:00,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:01,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:02,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:03,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:04,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:05,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:05,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:06,611 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:40:06,611 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:40:06,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:06,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:07,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:08,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:09,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:10,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:11,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:11,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:12,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:13,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:14,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:15,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:16,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:16,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:17,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:18,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:19,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:20,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:21,970 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:21,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:22,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:22,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:23,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:24,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:25,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:26,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:27,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:27,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:28,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:29,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:30,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:31,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:32,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:32,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:33,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:34,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:35,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:36,612 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:40:36,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:36,971 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:37,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:38,049 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:38,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:39,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:40,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:41,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:42,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:43,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:43,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:44,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:45,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:46,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:47,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:48,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:48,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:49,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:50,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:51,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:40:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:40:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:52,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:53,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:54,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:54,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:55,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:56,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:57,985 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:58,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:40:59,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:40:59,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:00,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:01,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:02,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:03,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:04,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:04,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:05,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:06,612 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:41:06,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:06,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:07,986 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:08,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:09,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:10,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:10,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:11,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:12,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:13,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:14,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:15,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:15,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:16,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:17,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:18,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:19,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:20,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:21,971 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:21,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:22,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:22,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:23,989 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:24,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:25,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:26,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:27,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:27,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:28,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:29,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:30,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:31,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:32,990 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:33,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:33,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:34,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:35,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:36,614 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:41:36,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:36,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:37,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:38,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:39,110 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:39,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:40,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:41,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:42,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:43,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:44,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:44,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:45,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:46,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:47,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:48,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:49,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:50,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:50,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:51,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:41:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:41:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:52,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:53,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:54,993 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:55,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:56,051 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:41:56,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:57,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:58,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:41:59,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:00,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:01,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:01,994 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:02,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:03,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:04,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:05,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:06,614 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:42:06,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:06,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:07,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:07,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:08,995 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:09,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:10,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:11,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:12,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:12,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:13,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:14,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:15,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:16,996 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:17,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:17,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:18,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:19,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:20,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:21,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:21,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:22,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:23,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:23,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:24,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:25,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:26,997 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:27,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:28,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:28,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:29,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:30,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:31,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:32,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:33,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:33,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:34,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:35,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:36,615 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:42:36,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:36,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:37,998 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:38,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:39,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:39,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:40,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:41,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:42,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:43,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:44,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:44,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:45,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:47,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:48,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:49,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:49,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:50,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:51,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:51,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:42:51,972 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:42:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:53,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:54,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:55,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:55,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:42:56,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:57,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:58,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:42:59,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:00,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:00,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:01,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:02,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:03,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:04,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:05,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:05,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:06,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:06,616 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:43:06,972 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:06,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:07,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:08,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:09,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:10,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:11,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:11,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:12,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:13,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:14,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:15,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:16,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:16,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:17,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:18,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:19,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:20,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:21,003 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:21,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:22,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:22,107 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:23,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:24,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:25,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:26,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:27,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:27,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:28,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:29,004 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:30,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:31,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:32,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:32,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:33,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:34,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:35,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:36,005 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:36,617 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:43:36,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:36,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:37,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:38,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:38,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:39,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:40,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:41,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:42,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:43,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:43,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:44,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:45,006 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:46,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:47,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:48,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:49,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:49,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:50,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:51,008 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:51,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:43:51,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:43:52,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:53,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:54,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:55,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:55,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:43:56,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:57,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:58,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:43:59,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:00,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:00,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:01,009 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:02,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:03,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:04,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:05,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:06,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:06,617 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:44:06,618 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:06,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:06,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:07,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:08,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:09,010 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:10,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:11,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:11,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:12,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:13,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:14,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:15,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:16,011 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:16,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:17,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:18,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:19,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:20,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:21,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:21,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:22,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:22,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:23,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:24,012 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:25,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:25,605 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:26,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:27,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:27,606 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:27,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:28,013 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:29,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:29,607 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:30,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:31,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:32,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:32,979 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:33,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:34,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:35,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:36,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:36,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:44:36,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:36,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:37,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:38,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:38,577 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:39,014 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:39,609 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:40,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:41,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:42,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:43,015 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:43,611 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:44:43,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:44,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:45,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:46,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:47,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:48,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:49,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:49,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:50,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:51,016 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:51,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:44:51,973 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:44:52,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:53,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:54,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:55,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:55,144 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:44:56,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:57,017 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:58,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:44:59,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:00,018 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:00,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:01,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:02,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:03,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:04,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:04,564 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:45:04,564 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:45:04,564 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 with policy now +2025-01-06 17:45:04,565 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:45:04,565 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:45:04,566 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:04,616 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos +2025-01-06 17:45:04,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media +2025-01-06 17:45:04,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:45:04,865 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/sqafg0jm-media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:05,019 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:05,151 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/l2dx4xh3-media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:45:05,617 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:06,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:06,579 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:06,618 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:45:06,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:06,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:07,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:08,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:09,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:09,618 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:10,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:11,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:11,619 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:11,937 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:12,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:13,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:13,619 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:14,020 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:15,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:16,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:16,980 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:17,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:18,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:19,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:20,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:21,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:21,973 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:21,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:22,021 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:22,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:23,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:24,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:25,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:26,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:26,624 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:45:26,870 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/tp0klld9-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:45:27,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:27,417 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:27,624 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:28,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:29,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:29,625 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:30,022 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:31,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:31,625 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:32,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:32,981 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:33,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:34,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:35,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:36,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:36,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:45:36,974 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:36,974 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:37,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:37,627 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:38,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:38,054 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:39,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:40,023 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:41,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:41,629 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:45:42,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:43,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:43,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:44,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:45,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:46,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:47,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:48,024 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:49,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:49,982 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:50,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:51,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:51,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:45:51,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:45:52,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:53,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:54,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:55,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:55,103 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:45:56,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:57,025 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:58,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:45:59,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:00,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:00,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:01,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:02,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:03,026 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:04,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:05,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:06,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:06,167 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:06,218 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:46:06,218 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 with policy now +2025-01-06 17:46:06,220 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:46:06,221 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:46:06,221 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:46:06,240 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:46:06,485 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/6h1a6y54-media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:46:06,920 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/j3pwg2pg-media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:46:06,977 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:06,977 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:07,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:07,637 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:08,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:09,027 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:10,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:11,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:11,638 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:11,889 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:12,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:13,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:13,639 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:14,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:15,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:16,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:16,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:17,029 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:18,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:19,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:20,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:21,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:21,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:21,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:22,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:22,124 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:23,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:24,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:25,030 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:26,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:27,031 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:27,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:28,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:29,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:30,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:31,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:32,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:33,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:33,649 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:46:33,651 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:33,790 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:46:33,790 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:46:33,791 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:33,791 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:46:33,924 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/5027jja0-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:46:34,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:34,649 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:46:35,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:35,650 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:36,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:36,619 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:46:36,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:36,975 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:37,032 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:37,650 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:38,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:39,033 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:39,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:40,034 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:41,034 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:42,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:43,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:44,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:44,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:45,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:46,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:47,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:48,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:49,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:50,035 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:50,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:51,036 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:51,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:46:51,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:46:52,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:53,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:54,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:55,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:55,656 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:46:56,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:56,854 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:46:57,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:58,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:59,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:46:59,658 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:00,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:01,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:01,987 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:02,037 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:03,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:04,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:05,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:06,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:06,620 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:47:06,975 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:06,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:07,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:07,098 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:08,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:09,038 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:10,039 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:11,039 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:12,042 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:12,179 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:12,515 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:47:12,515 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 with policy now +2025-01-06 17:47:12,515 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:47:12,516 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:47:12,516 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:47:12,525 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:12,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:47:12,791 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/st0fd33d-media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:13,042 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:13,059 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/y6u3pzub-media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:47:13,662 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:14,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:15,045 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:16,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:17,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:17,664 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:17,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:18,046 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:19,043 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:19,664 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:20,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:21,044 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:21,988 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:21,988 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:22,045 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:23,046 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:23,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:23,615 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:47:23,615 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:47:23,615 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:47:23,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:24,047 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:25,048 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:25,666 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:26,051 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:27,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:27,667 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:28,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:28,117 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:29,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:29,668 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:30,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:31,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:32,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:33,057 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:33,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:34,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:35,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:36,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:36,621 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:47:36,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:36,976 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:37,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:38,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:39,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:39,138 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:40,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:41,058 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:41,672 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:42,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:43,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:44,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:44,185 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:45,059 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:45,674 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:46,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:47,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:47,675 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:47:48,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:49,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:49,991 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:50,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:51,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:51,976 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:47:51,977 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:47:52,060 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:53,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:54,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:55,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:55,117 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:47:56,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:57,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:58,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:47:59,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:00,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:00,992 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:01,061 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:02,070 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:02,815 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:48:02,815 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 with policy now +2025-01-06 17:48:02,819 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:48:02,844 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:48:02,844 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:48:02,869 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:48:03,064 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:03,076 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/y7qlbzk4-media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:03,680 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:48:03,976 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/5jr7o643-media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:48:04,076 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:05,077 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:06,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:06,471 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:06,622 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:48:06,983 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:06,983 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:07,079 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:07,681 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:08,081 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:09,080 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:10,078 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:11,079 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:11,683 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:12,001 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:12,083 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:13,080 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:14,085 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:15,081 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:16,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:17,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:17,082 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:18,083 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:19,086 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:20,086 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:21,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:21,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:48:21,692 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:48:21,692 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:48:21,693 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:48:21,965 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/vxbri0xf-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:48:21,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:21,985 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:22,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:22,140 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:22,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:48:23,096 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:23,686 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:24,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:25,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:25,687 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:26,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:27,088 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:27,688 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:48:28,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:28,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:29,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:30,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:31,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:32,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:33,089 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:33,999 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:34,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:35,090 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:36,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:36,622 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:48:36,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:36,978 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:37,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:38,092 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:39,077 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:39,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:40,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:41,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:42,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:43,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:44,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:45,000 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:45,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:46,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:47,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:48,093 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:49,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:50,002 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:50,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:51,094 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:51,978 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:48:51,979 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:48:52,095 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:53,095 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:54,096 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:55,048 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:48:55,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:56,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:57,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:58,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:59,097 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:48:59,699 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:00,102 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:00,567 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:01,120 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:01,177 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:01,177 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 with policy now +2025-01-06 17:49:01,191 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:01,191 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:01,191 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:01,212 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:01,441 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/uhz293fi-media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:01,700 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:01,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval +2025-01-06 17:49:01,992 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/nkocmcjf-media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:02,109 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:03,115 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:03,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:04,116 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:05,103 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:05,701 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:05,897 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:06,115 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:06,623 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:49:06,984 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:49:06,984 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:49:07,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:08,111 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:09,111 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:09,578 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:49:09,578 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:09,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media +2025-01-06 17:49:09,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table +2025-01-06 17:49:09,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval +2025-01-06 17:49:09,787 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:09,890 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/mc9dnymz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:10,042 INFO wandb-upload_2:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptzqc3sdw +2025-01-06 17:49:10,104 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:10,137 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:49:10,177 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:49:10,480 INFO SenderThread:2093159 [sender.py:send_request_log_artifact():1525] logged artifact run-1jous172-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTE2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxNDc1Nw==', 'latestArtifact': None}} +2025-01-06 17:49:10,481 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:10,481 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 17:49:10,481 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:10,482 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,482 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,482 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:10,483 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,486 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send():379] send: telemetry +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,487 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,487 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,487 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,488 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,488 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,488 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,488 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,489 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,489 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,489 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,490 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,490 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,490 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,490 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,491 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,491 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,491 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,491 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:10,492 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:10,492 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:49:10,703 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:10,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval +2025-01-06 17:49:10,728 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/4kkpa2oc-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:10,854 INFO wandb-upload_2:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmrxdjnzp +2025-01-06 17:49:10,930 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/pnjp4gzq-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:11,104 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:11,247 INFO SenderThread:2093159 [sender.py:send_request_log_artifact():1525] logged artifact run-1jous172-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTE5Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxNDc2MQ==', 'latestArtifact': None}} +2025-01-06 17:49:11,247 DEBUG SenderThread:2093159 [sender.py:send():379] send: files +2025-01-06 17:49:11,247 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-06 17:49:11,248 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:11,248 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:11,248 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:11,248 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:11,497 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/wyeihir3-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:11,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:11,704 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:12,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:13,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:14,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:15,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:16,105 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:16,578 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:16,621 DEBUG SenderThread:2093159 [sender.py:send():379] send: exit +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():588] handling runtime: 610 +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:16,622 INFO SenderThread:2093159 [sender.py:send_exit():594] send defer +2025-01-06 17:49:16,622 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,622 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:49:16,622 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:49:16,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:49:16,623 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:49:16,623 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:49:16,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:49:16,623 INFO HandlerThread:2093159 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:49:16,623 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:49:16,624 DEBUG SystemMonitor:2093159 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:49:16,624 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:49:16,624 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:49:16,694 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:49:16,694 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:49:16,695 INFO HandlerThread:2093159 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:49:16,695 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,695 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:49:16,695 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:49:16,695 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,695 DEBUG SenderThread:2093159 [sender.py:send():379] send: stats +2025-01-06 17:49:16,695 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:49:16,696 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:16,696 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:49:16,696 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:49:16,696 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:16,696 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:49:16,706 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:17,706 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:22,758 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:49:22,759 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,759 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:49:22,759 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send():379] send: history +2025-01-06 17:49:22,760 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:49:22,760 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,760 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:22,760 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:49:22,761 DEBUG SenderThread:2093159 [sender.py:send():379] send: summary +2025-01-06 17:49:22,764 INFO SenderThread:2093159 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:49:22,764 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:49:22,765 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,765 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:49:22,765 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,765 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:49:22,768 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:49:22,906 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:49:22,906 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:22,906 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:49:22,906 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:22,906 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:49:23,623 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:23,709 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:23,709 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:49:25,145 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:49:25,145 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:25,145 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:25,145 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:49:25,146 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:25,146 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:49:25,146 INFO SenderThread:2093159 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:49:25,148 INFO SenderThread:2093159 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:49:25,150 INFO SenderThread:2093159 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:49:25,160 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:49:25,161 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:25,161 DEBUG SenderThread:2093159 [sender.py:send():379] send: artifact +2025-01-06 17:49:25,161 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:49:25,620 INFO wandb-upload_2:2093159 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwscw5le0 +2025-01-06 17:49:25,620 INFO wandb-upload_1:2093159 [upload_job.py:push():85] Skipped uploading /tmp/tmp3ojk991x/wandb-job.json +2025-01-06 17:49:25,624 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:25,710 INFO Thread-12 :2093159 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:25,726 INFO wandb-upload_0:2093159 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnh0s_z63 +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTk1OTYxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'versionIndex': 424}}} +2025-01-06 17:49:26,354 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:49:26,354 INFO SenderThread:2093159 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml config.yaml +2025-01-06 17:49:26,711 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json wandb-summary.json +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt requirements.txt +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:26,712 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log output.log +2025-01-06 17:49:26,715 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/diff.patch diff.patch +2025-01-06 17:49:26,716 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 media/videos/eval/3_eval_3_77698069b89e74b81013.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 media/videos/eval/2_eval_2_c482b456ff56ddc8a1e1.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 media/videos/eval/0_eval_0_1f9e236adbaeed571af3.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 media/videos/eval/4_eval_4_254f22dd2caf14ce117c.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 media/videos/eval/1_eval_1_04037a9f5a7dfcbf716d.mp4 +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-06 17:49:26,719 INFO SenderThread:2093159 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 17:49:26,720 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:49:26,720 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:26,722 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:26,722 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:49:26,722 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:26,722 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:49:26,722 INFO SenderThread:2093159 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:49:26,897 INFO wandb-upload_1:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/wandb-summary.json +2025-01-06 17:49:26,912 INFO wandb-upload_2:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/config.yaml +2025-01-06 17:49:26,946 INFO wandb-upload_0:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/requirements.txt +2025-01-06 17:49:26,974 INFO wandb-upload_4:2093159 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/files/output.log +2025-01-06 17:49:26,995 INFO wandb-upload_3:2093159 [upload_job.py:push():130] Uploaded file /tmp/tmpdbd0smeswandb/ie0stzsg-events.out.tfevents.1736203146.leonmkim-ROG-Strix-G15CS-G15CS.2092961.0 +2025-01-06 17:49:27,195 INFO Thread-11 (_thread_body):2093159 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:49:27,195 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,195 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:49:27,196 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,196 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:49:27,196 INFO SenderThread:2093159 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:49:27,197 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,197 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:49:27,197 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:49:27,197 INFO SenderThread:2093159 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:49:27,407 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,407 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:49:27,407 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:49:27,407 INFO SenderThread:2093159 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:49:27,407 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:49:27,407 INFO HandlerThread:2093159 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send():379] send: final +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send():379] send: footer +2025-01-06 17:49:27,408 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: defer +2025-01-06 17:49:27,408 INFO SenderThread:2093159 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:49:27,408 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:27,409 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:49:27,409 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:49:27,410 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:49:27,411 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:49:27,413 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:49:27,469 DEBUG SenderThread:2093159 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:49:27,470 INFO MainThread:2093159 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:49:27,471 DEBUG HandlerThread:2093159 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:49:27,471 INFO HandlerThread:2093159 [handler.py:finish():882] shutting down handler +2025-01-06 17:49:28,409 INFO WriterThread:2093159 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [sender.py:finish():1615] shutting down sender +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:49:28,470 INFO SenderThread:2093159 [file_pusher.py:join():175] waiting for file pusher diff --git a/1jous172/wandb/run-20250106_173906-1jous172/logs/debug.log b/1jous172/wandb/run-20250106_173906-1jous172/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e3492409c088702263f5689db87190dd05bd27e9 --- /dev/null +++ b/1jous172/wandb/run-20250106_173906-1jous172/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Configure stats pid to 2092961 +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/settings +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/logs/debug.log +2025-01-06 17:39:06,222 INFO MainThread:2092961 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/wandb/run-20250106_173906-1jous172/logs/debug-internal.log +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172'} +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():619] starting backend +2025-01-06 17:39:06,223 INFO MainThread:2092961 [wandb_init.py:init():623] setting up manager +2025-01-06 17:39:06,227 INFO MainThread:2092961 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:39:06,229 INFO MainThread:2092961 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:39:06,240 INFO MainThread:2092961 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:39:06,249 INFO MainThread:2092961 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:39:06,532 INFO MainThread:2092961 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:39:06,596 INFO MainThread:2092961 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:39:06,597 INFO MainThread:2092961 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:39:06,970 INFO MainThread:2092961 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:39:06,971 INFO MainThread:2092961 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:39:06,972 INFO MainThread:2092961 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/1jous172/tb, True +2025-01-06 17:39:10,618 INFO MainThread:2092961 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:49:28,471 WARNING MsgRouterThr:2092961 [router.py:message_loop():77] message_loop has been closed diff --git a/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb b/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0645244d43cb27a11d329766647bbc65f2597dc2 Binary files /dev/null and b/1jous172/wandb/run-20250106_173906-1jous172/run-1jous172.wandb differ diff --git a/2izwx0i7/.hydra/config.yaml b/2izwx0i7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/2izwx0i7/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2izwx0i7/.hydra/hydra.yaml b/2izwx0i7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b489105cac767841ea8d9f8371582075ae4566d --- /dev/null +++ b/2izwx0i7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/2izwx0i7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2izwx0i7/.hydra/overrides.yaml b/2izwx0i7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2izwx0i7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2izwx0i7/eval_policy.log b/2izwx0i7/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..0b8c769eee6486c4ab2d5ac5cacb248540c6eb8b --- /dev/null +++ b/2izwx0i7/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 20:21:14,581][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:21:14,584][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 20:21:15,758][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/5ux8hvp6/episode_rosbags/episode_1_2025-01-07-20-31-22.bag b/5ux8hvp6/episode_rosbags/episode_1_2025-01-07-20-31-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..4da629e2c27599e8ade8f73ac447176d585a7fd7 --- /dev/null +++ b/5ux8hvp6/episode_rosbags/episode_1_2025-01-07-20-31-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091e13ae5f38d0b81ea14ff8a1cc83466d1f2e59bc7588ad9cbbc51def41b47e +size 1172468665 diff --git a/5ux8hvp6/episode_rosbags/episode_3_2025-01-07-20-32-29.bag b/5ux8hvp6/episode_rosbags/episode_3_2025-01-07-20-32-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..4a2b9c0e5fe496e2f28a8c3f1987e387a3524f9d --- /dev/null +++ b/5ux8hvp6/episode_rosbags/episode_3_2025-01-07-20-32-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e4a4736fcdf72cf2275c7fa6e3db9bc85244ec7af73852e30cbfff934e135f +size 1044571883 diff --git a/8m5srpvj/episode_rosbags/episode_1_2025-01-06-20-22-24.bag b/8m5srpvj/episode_rosbags/episode_1_2025-01-06-20-22-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..ae9b1d29cfb4ff912722174bb82478318216af9b --- /dev/null +++ b/8m5srpvj/episode_rosbags/episode_1_2025-01-06-20-22-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30616378db91c602551e5cf551da203cdb72961a11884d492f3a9377c3c0ff9c +size 2610591352 diff --git a/8m5srpvj/episode_rosbags/episode_2_2025-01-06-20-23-12.bag b/8m5srpvj/episode_rosbags/episode_2_2025-01-06-20-23-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..604b625ffbfa78e485058be148df0a073b7fdd6b --- /dev/null +++ b/8m5srpvj/episode_rosbags/episode_2_2025-01-06-20-23-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b49f048c1ee70f4445029d4daa681abbc03d0fc2f781c197501ab51c67ba0e +size 2597290039 diff --git a/8m5srpvj/episode_rosbags/episode_3_2025-01-06-20-24-01.bag b/8m5srpvj/episode_rosbags/episode_3_2025-01-06-20-24-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..77ecf419cd769228e734eede3549b1eaebb53929 --- /dev/null +++ b/8m5srpvj/episode_rosbags/episode_3_2025-01-06-20-24-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3daa8f35c85faa8bba783a79357256492e6a66ddb7b7c1ddacc2755e3888fe0 +size 2606093426 diff --git a/8m5srpvj/episode_rosbags/episode_4_2025-01-06-20-24-51.bag b/8m5srpvj/episode_rosbags/episode_4_2025-01-06-20-24-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..a428e03a82e1e6ffddf8fdd1a5a212450c8aa471 --- /dev/null +++ b/8m5srpvj/episode_rosbags/episode_4_2025-01-06-20-24-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f37e9b90a61507ed9005d55b7123be266ec83d4f80b7132f4c403f7bb5bcca84 +size 2602461509 diff --git a/fm3fz5a0/.hydra/config.yaml b/fm3fz5a0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/fm3fz5a0/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/fm3fz5a0/.hydra/hydra.yaml b/fm3fz5a0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69936f2471596ac3d36cf527640383bd7ba8e7dd --- /dev/null +++ b/fm3fz5a0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/fm3fz5a0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/fm3fz5a0/.hydra/overrides.yaml b/fm3fz5a0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/fm3fz5a0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/fm3fz5a0/eval_policy.log b/fm3fz5a0/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..9a029b9ff55d47045c59e060ca26fd7ed4985846 --- /dev/null +++ b/fm3fz5a0/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 20:26:23,190][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:26:23,193][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 20:26:26,429][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/guo9e7cv/episode_rosbags/episode_0_2025-01-06-22-28-49.bag b/guo9e7cv/episode_rosbags/episode_0_2025-01-06-22-28-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..5c6f2f4effe39abe8373d948d44751c44e9340b1 --- /dev/null +++ b/guo9e7cv/episode_rosbags/episode_0_2025-01-06-22-28-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edace10b287ed9f9601d08783301b475c793652c0eba6747da5cd43144f45049 +size 2619460886 diff --git a/guo9e7cv/episode_rosbags/episode_1_2025-01-06-22-29-43.bag b/guo9e7cv/episode_rosbags/episode_1_2025-01-06-22-29-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..1edab578727c2d77f03ebe0a78c75f607aa7fa0a --- /dev/null +++ b/guo9e7cv/episode_rosbags/episode_1_2025-01-06-22-29-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ddfcd352c7b73d7ce986ba3f007544406bd7ee5d60f6dfaf0fbe8cb3ea94dd +size 2600891397 diff --git a/guo9e7cv/episode_rosbags/episode_2_2025-01-06-22-30-32.bag b/guo9e7cv/episode_rosbags/episode_2_2025-01-06-22-30-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..7d3b4b37e4e117b9a4ab4cc5278c820bddc94d49 --- /dev/null +++ b/guo9e7cv/episode_rosbags/episode_2_2025-01-06-22-30-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3a0c8b06885055c2bc477868de46a03940f894dbcd6549df637e5f063b1b421 +size 2598044644 diff --git a/guo9e7cv/episode_rosbags/episode_3_2025-01-06-22-31-31.bag b/guo9e7cv/episode_rosbags/episode_3_2025-01-06-22-31-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..e6eda8af3cc29b537a566867846dc57602c0d729 --- /dev/null +++ b/guo9e7cv/episode_rosbags/episode_3_2025-01-06-22-31-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0cdba8ed3025bd71009f67298c4d8ed96f0865eefd8db9557d5a0b57f802fb +size 2610072049 diff --git a/h2bqbukm/episode_rosbags/episode_0_2025-01-06-17-50-15.bag b/h2bqbukm/episode_rosbags/episode_0_2025-01-06-17-50-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..8722b93afb4d0c6ff0075925363dc1c7a5a184c6 --- /dev/null +++ b/h2bqbukm/episode_rosbags/episode_0_2025-01-06-17-50-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0da1ce0ff45e4288d1a9432e4b6c0a93854ca87a132aeb764fec0ab898141728 +size 2605532022 diff --git a/h2bqbukm/episode_rosbags/episode_1_2025-01-06-17-51-12.bag b/h2bqbukm/episode_rosbags/episode_1_2025-01-06-17-51-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..47ac5101be16cbc5ef3fc7de89ed645481d997e2 --- /dev/null +++ b/h2bqbukm/episode_rosbags/episode_1_2025-01-06-17-51-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63fae5ed01ef9a06e014a1f1f5bba24f3813a3b67b07b6c6e751d20b1d46af07 +size 2611592366 diff --git a/hcdwvdao/episode_rosbags/episode_0_2025-01-07-18-06-41.bag b/hcdwvdao/episode_rosbags/episode_0_2025-01-07-18-06-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..e7391903ebd30903fcaec16be1940dc31419889f --- /dev/null +++ b/hcdwvdao/episode_rosbags/episode_0_2025-01-07-18-06-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f29aafb676eaadb42dc1cc24f11b35b2eae20b4ae137457adb15f73afe8bdae1 +size 2623546867 diff --git a/hqgv52wz/.hydra/config.yaml b/hqgv52wz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/hqgv52wz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hqgv52wz/.hydra/hydra.yaml b/hqgv52wz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e5f8e19c61497b1f86451b2d01cc72856810309 --- /dev/null +++ b/hqgv52wz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hqgv52wz/.hydra/overrides.yaml b/hqgv52wz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hqgv52wz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hqgv52wz/episode_rosbags/color_K.npy b/hqgv52wz/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hqgv52wz/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hqgv52wz/episode_rosbags/episode_1_2025-01-07-20-27-28.bag b/hqgv52wz/episode_rosbags/episode_1_2025-01-07-20-27-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..a2c967a3dc4937b2ae2745f78d8e3264f674bda0 --- /dev/null +++ b/hqgv52wz/episode_rosbags/episode_1_2025-01-07-20-27-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39f9cde84db89bdefcc8fe3a1b7f8f25987074bec970550f5bcbc44a579be1c +size 1098950966 diff --git a/hqgv52wz/episode_rosbags/episode_4_2025-01-07-20-29-14.bag b/hqgv52wz/episode_rosbags/episode_4_2025-01-07-20-29-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..0395c907ef599680819d218f01b7626987df5369 --- /dev/null +++ b/hqgv52wz/episode_rosbags/episode_4_2025-01-07-20-29-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:579a221063ccd4bc9f19daf0ab08c9a377d7b5037a4c13a181443202aab2b316 +size 959186108 diff --git a/hqgv52wz/eval_video/0_eval.mp4 b/hqgv52wz/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67a2f872136cfd2f8ced5a32e20c292bf5cb286b --- /dev/null +++ b/hqgv52wz/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e526e809cdb4cfa35c3756e18db66c7a38ef3cc6461ce73cbfe3ad96b3bf4687 +size 929214 diff --git a/hqgv52wz/eval_video/1_eval.mp4 b/hqgv52wz/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..863f7aa8f948368dfa166f1b761aae98af101857 --- /dev/null +++ b/hqgv52wz/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1581f04c24b05aab09e952850aee8625c23c735b9b2df63b4e49d1fbe9dc129d +size 699781 diff --git a/hqgv52wz/eval_video/2_eval.mp4 b/hqgv52wz/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c92ca9905f49ec9dbaae08f76deee7ed319c528b --- /dev/null +++ b/hqgv52wz/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5de1b0b33efa6561b330a874e5b8d78f26d6e8bdcef02860f3d208d1076cc407 +size 739840 diff --git a/hqgv52wz/eval_video/3_eval.mp4 b/hqgv52wz/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98d5a29c9806e8fd8eccdc75ae6e81ec51549b69 --- /dev/null +++ b/hqgv52wz/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e4626513bb4dce7d642b134321d535e3abcc41ad3424e832b9702d55f96e8d1 +size 723628 diff --git a/hqgv52wz/eval_video/4_eval.mp4 b/hqgv52wz/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e0929a7e13fc6627b6d6411c1c18fd16d6de0cd --- /dev/null +++ b/hqgv52wz/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bee9eeae8343a1ab035ac12783f0deadc84a25722e4b2430ecfc49b3e43469c2 +size 690584 diff --git a/hqgv52wz/tb/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 b/hqgv52wz/tb/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 new file mode 100644 index 0000000000000000000000000000000000000000..babf7334c605921d133a0c22e873cbdc763a3656 --- /dev/null +++ b/hqgv52wz/tb/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54511a7c337aad176bcf54ed0c78ae1c00f50b0ab9b7cfa8965f098e0a5ac0e3 +size 6519 diff --git a/hqgv52wz/wandb/debug-internal.log b/hqgv52wz/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a4bb01a11a76c323890d20104b2806723b2ffd14 --- /dev/null +++ b/hqgv52wz/wandb/debug-internal.log @@ -0,0 +1,693 @@ +2025-01-07 20:26:31,370 INFO StreamThr :2225663 [internal.py:wandb_internal():85] W&B internal server running at pid: 2225663, started at: 2025-01-07 20:26:31.370103 +2025-01-07 20:26:31,371 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:26:31,383 INFO WriterThread:2225663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb +2025-01-07 20:26:31,383 DEBUG SenderThread:2225663 [sender.py:send():379] send: header +2025-01-07 20:26:31,392 DEBUG SenderThread:2225663 [sender.py:send():379] send: run +2025-01-07 20:26:31,621 INFO SenderThread:2225663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files +2025-01-07 20:26:31,621 INFO SenderThread:2225663 [sender.py:_start_run_threads():1188] run started: hqgv52wz with start time 1736299591.37047 +2025-01-07 20:26:31,638 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:26:31,638 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:26:31,710 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:26:31,715 DEBUG HandlerThread:2225663 [system_info.py:__init__():26] System info init +2025-01-07 20:26:31,715 DEBUG HandlerThread:2225663 [system_info.py:__init__():41] System info init done +2025-01-07 20:26:31,715 INFO HandlerThread:2225663 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:26:31,715 INFO SystemMonitor:2225663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:26:31,716 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:26:31,716 INFO HandlerThread:2225663 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:26:31,716 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:26:31,717 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:26:31,718 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:26:31,718 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:26:31,730 DEBUG HandlerThread:2225663 [system_info.py:probe():152] Probing system +2025-01-07 20:26:31,732 DEBUG HandlerThread:2225663 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_info.py:probe():200] Probing system done +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:26:31.730676', 'startedAt': '2025-01-08T01:26:31.362073', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4130.287125000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4509.796, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.75, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 583.1936264038086}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:26:31,736 INFO HandlerThread:2225663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:26:31,737 INFO HandlerThread:2225663 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:26:31,737 DEBUG HandlerThread:2225663 [system_info.py:_save_code():44] Saving code +2025-01-07 20:26:31,738 DEBUG HandlerThread:2225663 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:26:31,738 DEBUG HandlerThread:2225663 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:26:31,807 DEBUG HandlerThread:2225663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:26:31,807 INFO HandlerThread:2225663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:26:31,810 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:26:32,044 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/11i60q7r-wandb-metadata.json +2025-01-07 20:26:32,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:26:32,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:26:32,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:26:32,064 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:26:32,064 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:32,066 INFO HandlerThread:2225663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/tb" + save: true +} +control { + relay_id: "a5cce033978c44bda72f9c7906da82e3" +} +_info { + stream_id: "hqgv52wz" +} + +2025-01-07 20:26:32,066 DEBUG HandlerThread:2225663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:26:32,090 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/4986bzvo-code/FISH/eval_robot.py +2025-01-07 20:26:32,091 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/z9v54q5z-diff.patch +2025-01-07 20:26:32,215 DEBUG SenderThread:2225663 [sender.py:send():379] send: telemetry +2025-01-07 20:26:32,215 DEBUG SenderThread:2225663 [sender.py:send():379] send: tbrecord +2025-01-07 20:26:32,216 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:26:32,216 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 with policy live +2025-01-07 20:26:32,403 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/hsu8l67p-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py +2025-01-07 20:26:32,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH +2025-01-07 20:26:32,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code +2025-01-07 20:26:33,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:34,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:34,622 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:35,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:36,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:36,274 DEBUG SenderThread:2225663 [sender.py:send():379] send: config +2025-01-07 20:26:37,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:37,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:38,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:38,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:39,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:40,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:41,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:42,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:42,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:42,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:43,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:44,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:44,625 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:45,085 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:46,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:46,626 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:47,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:26:47,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:26:47,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:47,173 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:48,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:48,626 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:49,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:50,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:50,627 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:51,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:52,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:53,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:53,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:54,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:55,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:56,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:57,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:58,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:58,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:59,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:00,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:01,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:02,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:02,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:02,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:03,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:03,143 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:03,630 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:27:04,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:05,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:06,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:07,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:08,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:09,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:09,325 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:10,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:10,632 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:10,800 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:27:10,800 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:27:10,800 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 with policy now +2025-01-07 20:27:10,801 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:27:10,801 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:27:10,802 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:27:11,051 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/371xivyu-media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:11,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:11,632 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:27:11,950 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/32qwmyh1-media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:12,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:12,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:13,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:14,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:14,514 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:15,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:16,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:16,634 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:17,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:17,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:17,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:18,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:18,635 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:19,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:20,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:20,267 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:21,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:22,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:23,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:24,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:25,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:25,637 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:27:26,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:26,129 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:26,638 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:27,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:28,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:28,638 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:29,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:30,076 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:30,639 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:31,076 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:31,718 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:27:31,719 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:27:31,719 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:32,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:32,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:32,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:33,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:34,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:35,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:36,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:37,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:37,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:38,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:39,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:40,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:41,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:42,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:43,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:43,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:44,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:44,643 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:44,997 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:27:44,997 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 with policy now +2025-01-07 20:27:45,000 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:27:45,000 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:27:45,000 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:27:45,001 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:27:45,079 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:45,229 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/d2r9fb7e-media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:27:45,931 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/yo4ec2f5-media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:46,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:46,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:47,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:47,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:47,109 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:47,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:48,083 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:48,212 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:49,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:50,083 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:50,645 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:51,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:52,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:52,646 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:53,085 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:54,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:54,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:55,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:56,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:57,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:58,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:59,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:59,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:00,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:01,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:01,720 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:28:02,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:02,064 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:02,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:03,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:04,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:04,454 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:04,455 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:04,455 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:04,456 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:04,650 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:04,650 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:04,904 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/a57159td-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:05,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:06,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:06,651 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:07,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:08,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:08,651 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:09,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:10,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:10,150 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:10,652 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:11,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:12,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:13,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:14,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:15,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:16,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:16,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:17,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:17,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:17,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:18,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:19,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:20,092 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:21,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:21,214 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:22,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:23,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:24,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:25,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:25,149 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:28:25,149 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:28:25,150 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 with policy now +2025-01-07 20:28:25,150 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:25,150 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:25,150 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:25,389 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/2wec489j-media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:25,657 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:25,657 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:25,658 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:28:25,932 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/beyohbr8-media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:26,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:26,658 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:27,075 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:27,100 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:28,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:29,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:30,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:30,659 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:31,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:31,720 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:28:32,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:32,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:32,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:32,186 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:32,660 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:33,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:34,101 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:35,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:36,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:37,103 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:38,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:38,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:38,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:39,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:39,272 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:39,272 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:39,272 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:39,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:40,102 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:40,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:41,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:42,107 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:43,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:43,477 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:44,101 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:44,665 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:45,102 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:46,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:46,666 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:47,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:47,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:47,109 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:48,103 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:49,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:49,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:50,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:50,667 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:51,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:52,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:53,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:54,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:54,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:55,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:56,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:57,107 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:58,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:59,108 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:59,336 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:59,641 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:28:59,641 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 with policy now +2025-01-07 20:28:59,644 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:28:59,645 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:59,645 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:59,645 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:59,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:59,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:28:59,671 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:28:59,881 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/crw42gz1-media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:29:00,111 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:00,147 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/0ye9h6mf-media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:29:00,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:01,111 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:01,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:29:02,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:02,067 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:02,113 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:03,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:04,122 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:04,672 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:05,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:05,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:06,112 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:06,673 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:07,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:08,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:08,673 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:09,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:10,122 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:10,162 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:10,310 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:10,310 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:10,311 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:10,674 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:10,674 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:10,956 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/1a4x1ic9-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:11,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:12,136 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:12,675 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:13,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:14,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:15,133 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:15,822 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:16,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:16,676 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:17,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:17,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:17,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:18,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:18,677 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:19,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:20,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:20,678 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:21,013 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:21,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:22,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:23,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:24,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:25,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:26,075 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:26,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:27,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:28,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:29,128 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:30,141 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:30,180 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:30,180 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 with policy now +2025-01-07 20:29:30,225 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:30,226 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:30,226 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:30,227 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:30,412 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/dec7konm-media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:29:30,954 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/n2xuqwoo-media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:31,141 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:31,227 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:31,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:29:32,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:32,074 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:32,142 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:32,682 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:33,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:34,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:35,142 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:36,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:36,683 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:36,744 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:37,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:38,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:38,684 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:39,149 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:40,156 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:41,159 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:42,089 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:42,156 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:43,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:43,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:43,978 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:29:43,978 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:29:44,115 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:44,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:44,497 INFO wandb-upload_2:2225663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6zb6h9q5 +2025-01-07 20:29:44,591 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:29:44,599 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table +2025-01-07 20:29:45,099 INFO SenderThread:2225663 [sender.py:send_request_log_artifact():1525] logged artifact run-hqgv52wz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4OTMwMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTgyMQ==', 'latestArtifact': None}} +2025-01-07 20:29:45,099 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:45,099 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,105 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,109 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,109 DEBUG SenderThread:2225663 [sender.py:send():379] send: telemetry +2025-01-07 20:29:45,109 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,109 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,110 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,110 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,110 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,110 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,111 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,111 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,111 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,111 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,112 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,112 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,113 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,113 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,113 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,113 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,114 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,114 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,114 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,114 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:29:45,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:45,283 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/wks8x9mk-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:45,291 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/m8oesxz9-media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:45,489 INFO wandb-upload_2:2225663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprko7p45n +2025-01-07 20:29:45,575 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/65pdq4qm-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:45,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:45,888 INFO SenderThread:2225663 [sender.py:send_request_log_artifact():1525] logged artifact run-hqgv52wz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4OTMxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTgyNA==', 'latestArtifact': None}} +2025-01-07 20:29:45,888 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:45,888 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json with policy now +2025-01-07 20:29:45,889 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,889 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,889 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:46,141 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/g5ergqc0-media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:46,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:46,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:46,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:47,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:47,068 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:47,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:47,227 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:47,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:29:48,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:49,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:50,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:51,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:52,155 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:53,080 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:53,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:54,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:55,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:56,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:57,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:58,080 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:58,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:59,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:00,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:01,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:01,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:30:02,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:30:02,068 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:30:02,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:03,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:03,207 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:04,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:05,155 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:05,413 DEBUG SenderThread:2225663 [sender.py:send():379] send: exit +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():588] handling runtime: 213 +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():594] send defer +2025-01-07 20:30:05,414 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,414 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:30:05,415 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:30:05,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:30:05,415 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:30:05,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:30:05,416 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:30:05,416 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:30:05,416 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:30:05,417 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:30:05,469 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:30:05,470 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,470 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:30:05,470 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:30:05,470 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:30:05,470 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:30:05,471 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,471 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:30:05,471 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:30:05,471 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,471 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:30:05,695 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:06,695 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:30:12,360 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,360 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:30:12,361 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:12,361 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:30:12,361 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:30:12,362 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:12,362 DEBUG SenderThread:2225663 [sender.py:send():379] send: summary +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:12,366 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:30:12,366 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,366 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:30:12,367 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:30:12,367 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:12,367 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,367 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:30:12,367 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:30:12,414 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:12,698 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:13,061 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:30:13,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:13,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,062 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:30:13,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,062 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:30:13,062 INFO SenderThread:2225663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:30:13,064 INFO SenderThread:2225663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:30:13,066 INFO SenderThread:2225663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:30:13,077 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:30:13,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,077 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:30:13,078 DEBUG SenderThread:2225663 [sender.py:send():379] send: artifact +2025-01-07 20:30:13,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:13,478 INFO SenderThread:2225663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:30:13,478 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,478 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:30:13,479 INFO SenderThread:2225663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:30:13,698 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml config.yaml +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json wandb-summary.json +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt requirements.txt +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log output.log +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch diff.patch +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:30:13,705 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:30:13,707 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:13,708 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,708 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:30:13,708 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,708 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:30:13,708 INFO SenderThread:2225663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:30:13,878 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:30:13,884 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:13,948 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/7cocznj6-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:30:13,949 INFO wandb-upload_3:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt +2025-01-07 20:30:13,957 INFO wandb-upload_4:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:14,157 INFO Thread-11 (_thread_body):2225663 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:30:14,158 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,158 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:30:14,158 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,158 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:30:14,158 INFO SenderThread:2225663 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:30:14,159 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:30:14,159 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,159 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:30:14,160 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,160 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:30:14,160 INFO SenderThread:2225663 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:30:14,387 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,387 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:30:14,387 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:30:14,388 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:30:14,388 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,388 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send():379] send: final +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send():379] send: footer +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,388 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:14,389 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:30:14,390 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:30:14,391 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:30:14,392 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:14,451 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:14,451 INFO MainThread:2225663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:30:14,452 INFO MainThread:2225663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:30:14,452 INFO MainThread:2225663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:30:14,452 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:30:14,452 INFO HandlerThread:2225663 [handler.py:finish():882] shutting down handler +2025-01-07 20:30:15,390 INFO WriterThread:2225663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [sender.py:finish():1615] shutting down sender +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [file_pusher.py:join():175] waiting for file pusher diff --git a/hqgv52wz/wandb/debug.log b/hqgv52wz/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..369faa9a2eec82b5fa3dd9c91c7b41c899536abf --- /dev/null +++ b/hqgv52wz/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Configure stats pid to 2225469 +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/settings +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug.log +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug-internal.log +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz'} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():619] starting backend +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():623] setting up manager +2025-01-07 20:26:31,369 INFO MainThread:2225469 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:26:31,370 INFO MainThread:2225469 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:26:31,382 INFO MainThread:2225469 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:26:31,390 INFO MainThread:2225469 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:26:31,637 INFO MainThread:2225469 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:26:31,706 INFO MainThread:2225469 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:26:31,706 INFO MainThread:2225469 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:26:32,065 INFO MainThread:2225469 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:26:32,066 INFO MainThread:2225469 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/tb, True +2025-01-07 20:26:36,273 INFO MainThread:2225469 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:30:15,453 WARNING MsgRouterThr:2225469 [router.py:message_loop():77] message_loop has been closed diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1078f5005b7a025a78d301f42b4055212e51367f --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736299591 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..4697b4ebb6e44a4ea64b46b459fc2cf4999dc3f0 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch @@ -0,0 +1,16 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1643b06 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,9 +138,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +-# wandb_run_id: '1003_0' ++wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..40301c31094d803759381b9434f4ee3fdaedd572 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[43, 55, 0.7818181818181819, 0.631233260009268, 0.8913406579903888, 0.6627486049803439, 0.881260324193281, 0.682857152038004, 0.8600708729798676, 0.6975152901910096, 0.851549930800223]]} \ No newline at end of file diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67a2f872136cfd2f8ced5a32e20c292bf5cb286b --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e526e809cdb4cfa35c3756e18db66c7a38ef3cc6461ce73cbfe3ad96b3bf4687 +size 929214 diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..863f7aa8f948368dfa166f1b761aae98af101857 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1581f04c24b05aab09e952850aee8625c23c735b9b2df63b4e49d1fbe9dc129d +size 699781 diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c92ca9905f49ec9dbaae08f76deee7ed319c528b --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5de1b0b33efa6561b330a874e5b8d78f26d6e8bdcef02860f3d208d1076cc407 +size 739840 diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98d5a29c9806e8fd8eccdc75ae6e81ec51549b69 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e4626513bb4dce7d642b134321d535e3abcc41ad3424e832b9702d55f96e8d1 +size 723628 diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5e0929a7e13fc6627b6d6411c1c18fd16d6de0cd --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bee9eeae8343a1ab035ac12783f0deadc84a25722e4b2430ecfc49b3e43469c2 +size 690584 diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d2c3ffecd8fe3cf39a901774967d246b3f952279 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736299596.274332]: resetting environment +[INFO] [1736299596.278578]: cleared current plan +[INFO] [1736299596.279438]: moving to home +[INFO] [1736299597.281259]: reached home +[INFO] [1736299597.283283]: reset action history +[INFO] [1736299599.992191]: environment reset +Starting episode 0 +[INFO] [1736299599.996815]: resetting environment +[INFO] [1736299600.001007]: cleared current plan +[INFO] [1736299600.002204]: moving to home +[INFO] [1736299601.005776]: reached home +[INFO] [1736299601.007709]: reset action history +[INFO] [1736299603.716035]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736299605.041585]: resetting environment +[INFO] [1736299605.048084]: cleared current plan +[INFO] [1736299605.049040]: moving to home +[INFO] [1736299606.050831]: reached home +[INFO] [1736299606.052361]: reset action history +[INFO] [1736299608.758442]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/episode_rosbags/episode_0_2025-01-07-20-26-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2225469.log +started roslaunch server http://158.130.50.37:41127/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2225772] +process[print_text-2]: started with pid [2225796] +process[rosbag_record-3]: started with pid [2225797] +started bagging! +For topic gripper_width: timestamp difference is 101470977 for nearest: 1736299628269339353 - target: 1736299628167868376 at idx 249 +For topic gripper_width: timestamp difference is 104198973 for nearest: 1736299629569372456 - target: 1736299629465173483 at idx 249 +preempting episode +[INFO] [1736299629.809932]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736299630.589532]: Stored episode 1. +[INFO] [1736299630.592336]: Saving video... +[INFO] [1736299630.783310]: Video saved! +Starting episode 1 +[INFO] [1736299630.804343]: resetting environment +[INFO] [1736299630.810572]: cleared current plan +[INFO] [1736299630.810864]: moving to home +[INFO] [1736299634.513271]: reached home +[INFO] [1736299634.516061]: reset action history +[INFO] [1736299637.224842]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299645.124778]: resetting environment +[INFO] [1736299645.128251]: cleared current plan +[INFO] [1736299645.128423]: moving to home +[INFO] [1736299646.129272]: reached home +[INFO] [1736299646.129695]: reset action history +[INFO] [1736299648.838912]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/episode_rosbags/episode_1_2025-01-07-20-27-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2225469.log +started roslaunch server http://158.130.50.37:43201/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2225919] +process[print_text-5]: started with pid [2225920] +process[rosbag_record-6]: started with pid [2225921] +started bagging! +preempting episode +[INFO] [1736299663.778546]: Storing episode... +[print_text-5] killing on exit[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736299664.531768]: Stored episode 2. +[INFO] [1736299664.532196]: Saving video... +[INFO] [1736299664.981453]: Video saved! +Starting episode 2 +[INFO] [1736299665.000800]: resetting environment +[INFO] [1736299665.011933]: cleared current plan +[INFO] [1736299665.012790]: moving to home +[WARN] [1736299664.915972]: Plan exhausted +[WARN] [1736299664.956615]: Plan exhausted +[WARN] [1736299664.996012]: Plan exhausted +[INFO] [1736299668.626166]: reached home +[INFO] [1736299668.626822]: reset action history +[INFO] [1736299671.398986]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299683.968515]: resetting environment +[INFO] [1736299683.980313]: cleared current plan +[INFO] [1736299683.980591]: moving to home +[INFO] [1736299684.981767]: reached home +[INFO] [1736299684.983867]: reset action history +[INFO] [1736299687.704041]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/episode_rosbags/episode_2_2025-01-07-20-28-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2225469.log +started roslaunch server http://158.130.50.37:37475/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2226084] +process[print_text-8]: started with pid [2226085] +process[rosbag_record-9]: started with pid [2226086] +started bagging! +preempting episode +[INFO] [1736299704.060454]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736299704.888013]: Stored episode 3. +[INFO] [1736299704.891301]: Saving video... +[INFO] [1736299705.104828]: Video saved! +Starting episode 3 +[INFO] [1736299705.146075]: resetting environment +[INFO] [1736299705.157456]: cleared current plan +[INFO] [1736299705.157893]: moving to home +[INFO] [1736299708.760988]: reached home +[INFO] [1736299708.762736]: reset action history +[INFO] [1736299711.673237]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299718.605180]: resetting environment +[INFO] [1736299718.622917]: cleared current plan +[INFO] [1736299718.624877]: moving to home +[INFO] [1736299719.630726]: reached home +[INFO] [1736299719.634804]: reset action history +[INFO] [1736299722.435206]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/episode_rosbags/episode_3_2025-01-07-20-28-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2225469.log +started roslaunch server http://158.130.50.37:39595/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2226234] +process[print_text-11]: started with pid [2226235] +process[rosbag_record-12]: started with pid [2226236] +started bagging! +For topic gripper_width: timestamp difference is 111686230 for nearest: 1736299725269338608 - target: 1736299725157652378 at idx 45 +For topic gripper_width: timestamp difference is 113826743 for nearest: 1736299728002666942 - target: 1736299727888840199 at idx 80 +preempting episode +[INFO] [1736299738.514927]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736299739.332930]: Stored episode 4. +[INFO] [1736299739.338379]: Saving video... +[INFO] [1736299739.597896]: Video saved! +Starting episode 4 +[INFO] [1736299739.646372]: resetting environment +[INFO] [1736299739.668022]: cleared current plan +[INFO] [1736299739.671770]: moving to home +[INFO] [1736299743.076682]: reached home +[INFO] [1736299743.083247]: reset action history +[INFO] [1736299745.870766]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299750.202432]: resetting environment +[INFO] [1736299750.234992]: cleared current plan +[INFO] [1736299750.235523]: moving to home +[INFO] [1736299751.239044]: reached home +[INFO] [1736299751.244267]: reset action history +[INFO] [1736299754.066482]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/episode_rosbags/episode_4_2025-01-07-20-29-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2225469.log +started roslaunch server http://158.130.50.37:38261/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2226373] +process[print_text-14]: started with pid [2226374] +process[rosbag_record-15]: started with pid [2226375] +started bagging! +For topic gripper_width: timestamp difference is 107759583 for nearest: 1736299757969337332 - target: 1736299757861577749 at idx 51 +preempting episode +[INFO] [1736299769.042516]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736299769.858770]: Stored episode 5. +[INFO] [1736299769.872834]: Saving video... +[INFO] [1736299770.127518]: Video saved! +[INFO] [1736299770.184410]: resetting environment +[INFO] [1736299770.217015]: cleared current plan +[INFO] [1736299770.222050]: moving to home +[INFO] [1736299773.930564]: reached home +[INFO] [1736299773.944243]: reset action history +[INFO] [1736299776.743607]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..890f95c5346397b2cd482d475f880d3b89469ae1 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:26:31.730676", + "startedAt": "2025-01-08T01:26:31.362073", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4130.287125000001, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4509.796, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4502.75, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 583.1936264038086 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6ef7f09e46c7cff5bedccd5247b3c04f3f56bca8 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "e526e809cdb4cfa35c3756e18db66c7a38ef3cc6461ce73cbfe3ad96b3bf4687", "size": 929214, "path": "media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4"}, "global_step": 294, "_timestamp": 1736299784.4696674, "_runtime": 193.0991973876953, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "1581f04c24b05aab09e952850aee8625c23c735b9b2df63b4e49d1fbe9dc129d", "size": 699781, "path": "media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4"}, "eval/num_success": 0.0, "episode": 55.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "5de1b0b33efa6561b330a874e5b8d78f26d6e8bdcef02860f3d208d1076cc407", "size": 739840, "path": "media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "1e4626513bb4dce7d642b134321d535e3abcc41ad3424e832b9702d55f96e8d1", "size": 723628, "path": "media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "bee9eeae8343a1ab035ac12783f0deadc84a25722e4b2430ecfc49b3e43469c2", "size": 690584, "path": "media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://mr73mk3lz8donxkuwobxwoht3brc5iqpa2fh31ntwkhjzkdhnr919on6mc9tu65k04lvbjz2c7malsg82jspnewhhs7qb9qdjn23uxwq52837904c1h6alm32hms1vn8/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://mfgnieomq68zaka3toziwkg90919tpdhtpla3x1lazvrdnzb7fgf8hpnu3vcotcujyye9ltwvpbj8sabu54an0z4qon8dq9nfyzgeuk1p600hmmndxooeb2d7p4vi0lm:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 43, "total_num_episodes": 55, "total_success_rate": 0.7818181818181819, "total_success_rate_lb_0.01": 0.631233260009268, "total_success_rate_ub_0.01": 0.8913406579903888, "total_success_rate_lb_0.025": 0.6627486049803439, "total_success_rate_ub_0.025": 0.881260324193281, "total_success_rate_lb_0.05": 0.682857152038004, "total_success_rate_ub_0.05": 0.8600708729798676, "total_success_rate_lb_0.1": 0.6975152901910096, "total_success_rate_ub_0.1": 0.851549930800223, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c7323d29a79048302b4a49a7ac5a67e7710787c7ca54cf6965f9eddfa3939984", "size": 516, "artifact_path": "wandb-client-artifact://eoo8kibagb85san74cmxwou6mzet405f29prle7d3ld5ilpt2p7eijq070raivmhd5gkz0gouup3r2kauw99hdfnkdphcjvkdkxj2pg0tvlpett1puv0c3ktib6jsjpt/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://v9xyfknvpha1o7hcn3uwhvfbb5aa6m63yb36ohraikbi79y4wty6p5wpw4ozuqskucph1q0waa2va4cw8vtko0lz80vhbhacijo30x9p0q9vni70et0om7z9mw3yn2ge:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 213}, "eval/total_success_rate": 0.7818182110786438, "eval/total_success_rate_lb_0.01": 0.631233274936676, "eval/total_success_rate_ub_0.01": 0.8913406729698181, "eval/total_success_rate_lb_0.025": 0.6627485752105713, "eval/total_success_rate_ub_0.025": 0.881260335445404, "eval/total_success_rate_lb_0.05": 0.6828571557998657, "eval/total_success_rate_ub_0.05": 0.8600708842277527, "eval/total_success_rate_lb_0.1": 0.6975153088569641, "eval/total_success_rate_ub_0.1": 0.8515499234199524} \ No newline at end of file diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug-internal.log b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a4bb01a11a76c323890d20104b2806723b2ffd14 --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug-internal.log @@ -0,0 +1,693 @@ +2025-01-07 20:26:31,370 INFO StreamThr :2225663 [internal.py:wandb_internal():85] W&B internal server running at pid: 2225663, started at: 2025-01-07 20:26:31.370103 +2025-01-07 20:26:31,371 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:26:31,383 INFO WriterThread:2225663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb +2025-01-07 20:26:31,383 DEBUG SenderThread:2225663 [sender.py:send():379] send: header +2025-01-07 20:26:31,392 DEBUG SenderThread:2225663 [sender.py:send():379] send: run +2025-01-07 20:26:31,621 INFO SenderThread:2225663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files +2025-01-07 20:26:31,621 INFO SenderThread:2225663 [sender.py:_start_run_threads():1188] run started: hqgv52wz with start time 1736299591.37047 +2025-01-07 20:26:31,638 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:26:31,638 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:26:31,710 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:26:31,715 DEBUG HandlerThread:2225663 [system_info.py:__init__():26] System info init +2025-01-07 20:26:31,715 DEBUG HandlerThread:2225663 [system_info.py:__init__():41] System info init done +2025-01-07 20:26:31,715 INFO HandlerThread:2225663 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:26:31,715 INFO SystemMonitor:2225663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:26:31,716 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:26:31,716 INFO HandlerThread:2225663 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:26:31,716 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:26:31,717 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:26:31,718 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:26:31,718 INFO SystemMonitor:2225663 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:26:31,730 DEBUG HandlerThread:2225663 [system_info.py:probe():152] Probing system +2025-01-07 20:26:31,732 DEBUG HandlerThread:2225663 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_info.py:probe():200] Probing system done +2025-01-07 20:26:31,736 DEBUG HandlerThread:2225663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:26:31.730676', 'startedAt': '2025-01-08T01:26:31.362073', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4130.287125000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4509.796, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4502.75, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 583.1936264038086}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:26:31,736 INFO HandlerThread:2225663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:26:31,737 INFO HandlerThread:2225663 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:26:31,737 DEBUG HandlerThread:2225663 [system_info.py:_save_code():44] Saving code +2025-01-07 20:26:31,738 DEBUG HandlerThread:2225663 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:26:31,738 DEBUG HandlerThread:2225663 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:26:31,807 DEBUG HandlerThread:2225663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:26:31,807 INFO HandlerThread:2225663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:26:31,810 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:26:31,811 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:26:32,044 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/11i60q7r-wandb-metadata.json +2025-01-07 20:26:32,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:26:32,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:26:32,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:26:32,064 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:26:32,064 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:32,066 INFO HandlerThread:2225663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/tb" + save: true +} +control { + relay_id: "a5cce033978c44bda72f9c7906da82e3" +} +_info { + stream_id: "hqgv52wz" +} + +2025-01-07 20:26:32,066 DEBUG HandlerThread:2225663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:26:32,090 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/4986bzvo-code/FISH/eval_robot.py +2025-01-07 20:26:32,091 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/z9v54q5z-diff.patch +2025-01-07 20:26:32,215 DEBUG SenderThread:2225663 [sender.py:send():379] send: telemetry +2025-01-07 20:26:32,215 DEBUG SenderThread:2225663 [sender.py:send():379] send: tbrecord +2025-01-07 20:26:32,216 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:26:32,216 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 with policy live +2025-01-07 20:26:32,403 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/hsu8l67p-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json +2025-01-07 20:26:32,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py +2025-01-07 20:26:32,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH +2025-01-07 20:26:32,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code +2025-01-07 20:26:33,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:34,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:34,622 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:35,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:36,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:36,274 DEBUG SenderThread:2225663 [sender.py:send():379] send: config +2025-01-07 20:26:37,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:37,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:38,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:38,623 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:39,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:40,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:41,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:42,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:42,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:42,624 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:43,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:44,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:44,625 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:45,085 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:46,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:46,626 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:47,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:26:47,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:26:47,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:47,173 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:48,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:48,626 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:49,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:50,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:50,627 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:26:51,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:52,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:53,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:53,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:54,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:55,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:56,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:57,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:58,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:58,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:59,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:00,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:01,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:02,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:02,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:02,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:03,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:03,143 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:03,630 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:27:04,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:05,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:06,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:07,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:08,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:09,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:09,325 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:10,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:10,632 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:10,800 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:27:10,800 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:27:10,800 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 with policy now +2025-01-07 20:27:10,801 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:27:10,801 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:27:10,802 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:27:11,051 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/371xivyu-media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:11,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:11,632 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media +2025-01-07 20:27:11,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:27:11,950 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/32qwmyh1-media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:27:12,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:12,633 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:13,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:14,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:14,514 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:15,070 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:16,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:16,634 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:17,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:17,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:17,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:18,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:18,635 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:19,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:20,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:20,267 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:21,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:22,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:23,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:24,072 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:25,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:25,637 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:27:26,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:26,129 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:26,638 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:27,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:28,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:28,638 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:29,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:30,076 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:30,639 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:31,076 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:31,718 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:27:31,719 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:27:31,719 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:32,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:32,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:32,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:33,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:34,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:35,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:36,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:37,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:37,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:38,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:39,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:40,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:41,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:42,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:43,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:43,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:44,078 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:44,643 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:44,997 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:27:44,997 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 with policy now +2025-01-07 20:27:45,000 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:27:45,000 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:27:45,000 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:27:45,001 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:27:45,079 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:45,229 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/d2r9fb7e-media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:45,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:27:45,931 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/yo4ec2f5-media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:27:46,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:46,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:47,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:27:47,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:27:47,109 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:47,644 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:48,083 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:48,212 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:49,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:50,083 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:50,645 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:51,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:52,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:52,646 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:27:53,085 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:54,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:54,084 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:55,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:56,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:57,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:58,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:27:59,069 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:27:59,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:00,088 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:01,086 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:01,720 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:28:02,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:02,064 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:02,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:03,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:04,087 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:04,454 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:04,455 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:04,455 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:04,456 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:04,650 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:04,650 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:04,904 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/a57159td-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:05,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:06,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:06,651 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:07,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:08,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:08,651 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:09,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:10,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:10,150 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:10,652 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:11,090 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:12,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:13,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:14,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:15,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:16,068 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:16,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:17,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:17,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:17,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:18,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:19,091 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:20,092 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:21,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:21,214 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:22,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:23,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:24,093 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:25,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:25,149 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:28:25,149 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:28:25,150 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 with policy now +2025-01-07 20:28:25,150 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:25,150 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:25,150 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:25,389 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/2wec489j-media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:25,657 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:25,657 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:25,658 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:28:25,932 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/beyohbr8-media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:28:26,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:26,658 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:27,075 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:27,100 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:28,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:29,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:30,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:30,659 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:31,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:31,720 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:28:32,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:32,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:32,097 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:32,186 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:32,660 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:33,098 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:34,101 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:35,099 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:36,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:37,103 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:38,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:38,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:38,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:28:39,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:39,272 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:39,272 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:39,272 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:39,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:40,102 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:40,663 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:41,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:42,107 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:43,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:43,477 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:44,101 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:44,665 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:45,102 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:46,104 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:46,666 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:47,062 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:28:47,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:28:47,109 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:48,103 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:49,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:49,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:50,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:50,667 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:28:51,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:52,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:53,105 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:54,071 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:54,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:55,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:56,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:57,107 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:58,106 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:59,108 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:28:59,336 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:28:59,641 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:28:59,641 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 with policy now +2025-01-07 20:28:59,644 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:28:59,645 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:28:59,645 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:28:59,645 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:28:59,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:28:59,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:28:59,671 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:28:59,881 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/crw42gz1-media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:29:00,111 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:00,147 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/0ye9h6mf-media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:29:00,670 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:01,111 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:01,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:29:02,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:02,067 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:02,113 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:03,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:04,122 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:04,672 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:05,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:05,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:06,112 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:06,673 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:07,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:08,120 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:08,673 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:09,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:10,122 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:10,162 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:10,310 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:10,310 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:10,311 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:10,674 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:10,674 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:10,956 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/1a4x1ic9-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:11,121 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:12,136 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:12,675 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:13,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:14,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:15,133 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:15,822 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:16,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:16,676 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:17,063 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:17,063 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:17,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:18,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:18,677 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:19,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:20,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:20,678 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:21,013 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:21,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:22,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:23,126 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:24,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:25,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:26,075 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:26,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:27,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:28,127 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:29,128 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:30,141 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:30,180 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:30,180 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 with policy now +2025-01-07 20:29:30,225 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:30,226 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:30,226 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:30,227 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:30,412 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/dec7konm-media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:30,681 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval +2025-01-07 20:29:30,954 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/n2xuqwoo-media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:29:31,141 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:31,227 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:31,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:29:32,073 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:32,074 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:32,142 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:32,682 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:33,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:34,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:35,142 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:36,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:36,683 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:36,744 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:37,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:38,145 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:38,684 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:39,149 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:40,156 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:41,159 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:42,089 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:42,156 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:43,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:43,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:43,978 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:29:43,978 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:29:44,115 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:44,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:44,497 INFO wandb-upload_2:2225663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6zb6h9q5 +2025-01-07 20:29:44,591 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:29:44,599 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:44,686 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval +2025-01-07 20:29:44,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table +2025-01-07 20:29:45,099 INFO SenderThread:2225663 [sender.py:send_request_log_artifact():1525] logged artifact run-hqgv52wz-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4OTMwMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTgyMQ==', 'latestArtifact': None}} +2025-01-07 20:29:45,099 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:45,099 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,105 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,105 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,109 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,109 DEBUG SenderThread:2225663 [sender.py:send():379] send: telemetry +2025-01-07 20:29:45,109 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,109 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,110 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,110 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,110 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,110 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,111 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,111 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,111 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,111 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,112 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,112 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,112 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,113 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,113 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,113 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,113 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,114 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,114 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,114 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:45,114 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:29:45,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:45,283 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/wks8x9mk-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:45,291 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/m8oesxz9-media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:45,489 INFO wandb-upload_2:2225663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprko7p45n +2025-01-07 20:29:45,575 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/65pdq4qm-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:29:45,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:45,888 INFO SenderThread:2225663 [sender.py:send_request_log_artifact():1525] logged artifact run-hqgv52wz-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4OTMxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTgyNA==', 'latestArtifact': None}} +2025-01-07 20:29:45,888 DEBUG SenderThread:2225663 [sender.py:send():379] send: files +2025-01-07 20:29:45,888 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json with policy now +2025-01-07 20:29:45,889 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:29:45,889 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:29:45,889 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:29:46,141 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/g5ergqc0-media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:29:46,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:46,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:29:46,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:29:47,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:29:47,068 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:29:47,151 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:47,227 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:47,687 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:29:48,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:49,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:50,152 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:51,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:52,155 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:53,080 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:53,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:54,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:55,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:56,153 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:57,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:58,080 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:29:58,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:29:59,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:00,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:01,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:01,721 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:30:02,067 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:30:02,068 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:30:02,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:03,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:03,207 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:04,154 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:05,155 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:05,413 DEBUG SenderThread:2225663 [sender.py:send():379] send: exit +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():588] handling runtime: 213 +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:05,414 INFO SenderThread:2225663 [sender.py:send_exit():594] send defer +2025-01-07 20:30:05,414 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,414 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:30:05,415 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:30:05,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:30:05,415 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:30:05,415 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:30:05,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:30:05,415 INFO HandlerThread:2225663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:30:05,416 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:30:05,416 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:30:05,416 DEBUG SystemMonitor:2225663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:30:05,417 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:30:05,469 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:30:05,470 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,470 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:30:05,470 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:30:05,470 DEBUG SenderThread:2225663 [sender.py:send():379] send: stats +2025-01-07 20:30:05,470 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,470 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:30:05,471 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:05,471 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:30:05,471 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:30:05,471 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:05,471 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:30:05,695 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:06,695 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:12,359 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:30:12,360 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,360 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:30:12,361 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:12,361 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send():379] send: history +2025-01-07 20:30:12,361 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:30:12,361 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:30:12,362 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:12,362 DEBUG SenderThread:2225663 [sender.py:send():379] send: summary +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:30:12,366 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:30:12,366 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:30:12,366 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,366 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:30:12,367 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:30:12,367 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:30:12,367 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:12,367 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:30:12,367 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:12,367 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:30:12,414 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:12,698 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:13,061 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:30:13,061 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:13,061 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,062 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:30:13,062 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,062 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:30:13,062 INFO SenderThread:2225663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:30:13,064 INFO SenderThread:2225663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:30:13,066 INFO SenderThread:2225663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:30:13,077 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:30:13,077 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,077 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:30:13,078 DEBUG SenderThread:2225663 [sender.py:send():379] send: artifact +2025-01-07 20:30:13,415 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:13,478 INFO SenderThread:2225663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:30:13,478 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,478 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:30:13,479 INFO SenderThread:2225663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:30:13,698 INFO Thread-12 :2225663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml config.yaml +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json wandb-summary.json +2025-01-07 20:30:13,699 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt requirements.txt +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:30:13,702 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log output.log +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/diff.patch diff.patch +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 media/videos/eval/2_eval_2_5de1b0b33efa6561b330.mp4 +2025-01-07 20:30:13,703 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 media/videos/eval/3_eval_3_1e4626513bb4dce7d642.mp4 +2025-01-07 20:30:13,705 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 media/videos/eval/1_eval_1_1581f04c24b05aab09e9.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 media/videos/eval/4_eval_4_bee9eeae8343a1ab035a.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 media/videos/eval/0_eval_0_e526e809cdb4cfa35c37.mp4 +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json media/table/eval/total_success_rate_ci_6_c7323d29a79048302b4a.table.json +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:30:13,706 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:30:13,707 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:13,708 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:13,708 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:30:13,708 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:13,708 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:30:13,708 INFO SenderThread:2225663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:30:13,878 INFO wandb-upload_1:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/config.yaml +2025-01-07 20:30:13,884 INFO wandb-upload_2:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/wandb-summary.json +2025-01-07 20:30:13,948 INFO wandb-upload_0:2225663 [upload_job.py:push():130] Uploaded file /tmp/tmprkufu6ivwandb/7cocznj6-events.out.tfevents.1736299592.leonmkim-ROG-Strix-G15CS-G15CS.2225469.0 +2025-01-07 20:30:13,949 INFO wandb-upload_3:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/requirements.txt +2025-01-07 20:30:13,957 INFO wandb-upload_4:2225663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/files/output.log +2025-01-07 20:30:14,157 INFO Thread-11 (_thread_body):2225663 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:30:14,158 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,158 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:30:14,158 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,158 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:30:14,158 INFO SenderThread:2225663 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:30:14,159 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:30:14,159 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,159 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:30:14,160 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,160 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:30:14,160 INFO SenderThread:2225663 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:30:14,387 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,387 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:30:14,387 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,387 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:30:14,388 INFO SenderThread:2225663 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:30:14,388 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:30:14,388 INFO HandlerThread:2225663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send():379] send: final +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send():379] send: footer +2025-01-07 20:30:14,388 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: defer +2025-01-07 20:30:14,388 INFO SenderThread:2225663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:14,389 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:30:14,389 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:30:14,390 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:30:14,391 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:30:14,392 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:30:14,451 DEBUG SenderThread:2225663 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:30:14,451 INFO MainThread:2225663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:30:14,452 INFO MainThread:2225663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:30:14,452 INFO MainThread:2225663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:30:14,452 DEBUG HandlerThread:2225663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:30:14,452 INFO HandlerThread:2225663 [handler.py:finish():882] shutting down handler +2025-01-07 20:30:15,390 INFO WriterThread:2225663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [sender.py:finish():1615] shutting down sender +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:30:15,451 INFO SenderThread:2225663 [file_pusher.py:join():175] waiting for file pusher diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug.log b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..369faa9a2eec82b5fa3dd9c91c7b41c899536abf --- /dev/null +++ b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Configure stats pid to 2225469 +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/settings +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:26:31,364 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug.log +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/logs/debug-internal.log +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz'} +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():619] starting backend +2025-01-07 20:26:31,365 INFO MainThread:2225469 [wandb_init.py:init():623] setting up manager +2025-01-07 20:26:31,369 INFO MainThread:2225469 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:26:31,370 INFO MainThread:2225469 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:26:31,382 INFO MainThread:2225469 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:26:31,390 INFO MainThread:2225469 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:26:31,637 INFO MainThread:2225469 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:26:31,706 INFO MainThread:2225469 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:26:31,706 INFO MainThread:2225469 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:26:32,062 INFO MainThread:2225469 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:26:32,065 INFO MainThread:2225469 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:26:32,066 INFO MainThread:2225469 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hqgv52wz/tb, True +2025-01-07 20:26:36,273 INFO MainThread:2225469 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:30:15,453 WARNING MsgRouterThr:2225469 [router.py:message_loop():77] message_loop has been closed diff --git a/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e6bd98dfbc4d16095efef96d3a1bc108294e5123 Binary files /dev/null and b/hqgv52wz/wandb/run-20250107_202631-hqgv52wz/run-hqgv52wz.wandb differ diff --git a/hxim6a4m/.hydra/config.yaml b/hxim6a4m/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/hxim6a4m/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/hxim6a4m/.hydra/hydra.yaml b/hxim6a4m/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd30a8be3c1259f6af958e652103511e68415f33 --- /dev/null +++ b/hxim6a4m/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/hxim6a4m/.hydra/overrides.yaml b/hxim6a4m/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/hxim6a4m/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/hxim6a4m/episode_rosbags/aligned_depth_to_color_K.npy b/hxim6a4m/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hxim6a4m/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hxim6a4m/episode_rosbags/cam_tf_world.npy b/hxim6a4m/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hxim6a4m/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hxim6a4m/episode_rosbags/color_K.npy b/hxim6a4m/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hxim6a4m/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hxim6a4m/episode_rosbags/depth_K.npy b/hxim6a4m/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hxim6a4m/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hxim6a4m/eval_robot.log b/hxim6a4m/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..5b3285a814a98497835d0533a926b810f4e0cb31 --- /dev/null +++ b/hxim6a4m/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:34:23,686][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:34:23,689][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hxim6a4m/eval_video/0_eval.mp4 b/hxim6a4m/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0a49b792e8763d0cf3dc671f4b28b0e97f03757 --- /dev/null +++ b/hxim6a4m/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ff36b24aad6fe917e51fa7450a454bb835a8553386e2cdcfd25661ee99a59e +size 1414528 diff --git a/hxim6a4m/eval_video/1_eval.mp4 b/hxim6a4m/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..59db9d0af910e01e8eb2fdb534a1ca665b3fb861 --- /dev/null +++ b/hxim6a4m/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51fb3af7bcef5065307e5065de0b4e80b112e4bfb5e31eab835d14079a4b9e55 +size 1460297 diff --git a/hxim6a4m/eval_video/2_eval.mp4 b/hxim6a4m/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d12d876a86d2ea59031e5dd6deb872d84a4b4a96 --- /dev/null +++ b/hxim6a4m/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3f7d823766e09778f611008af23da5576adbf3900f797a867398b10a45c0ff +size 1498734 diff --git a/hxim6a4m/eval_video/3_eval.mp4 b/hxim6a4m/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8169bf2f89034830d966a5bdd81811f566c73e32 --- /dev/null +++ b/hxim6a4m/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f961b9a58ac3c44eab41a31a18131c71e967e8208a35af5b482ced5f81d5b87a +size 1430092 diff --git a/hxim6a4m/eval_video/4_eval.mp4 b/hxim6a4m/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbc6c5300e6556f4e6f531ba1e1ca1a05416f791 --- /dev/null +++ b/hxim6a4m/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5fec9c26b35ac2c23952b6d8f2b84c13f5bb9670b89849d5d6203bcb34283e +size 1453160 diff --git a/hxim6a4m/tb/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 b/hxim6a4m/tb/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 new file mode 100644 index 0000000000000000000000000000000000000000..51ee097af327fed9ef133be98010bf8fd1a00f3b --- /dev/null +++ b/hxim6a4m/tb/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6749a7b2fc211becab3a859b65dfe9cb9dc75b336dc61e76f8623503462ae607 +size 6539 diff --git a/hxim6a4m/wandb/debug-internal.log b/hxim6a4m/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..484a530a02b853878a7131f9254e0466da5640d6 --- /dev/null +++ b/hxim6a4m/wandb/debug-internal.log @@ -0,0 +1,833 @@ +2025-01-06 22:34:29,441 INFO StreamThr :2127250 [internal.py:wandb_internal():85] W&B internal server running at pid: 2127250, started at: 2025-01-06 22:34:29.440912 +2025-01-06 22:34:29,442 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:34:29,453 INFO WriterThread:2127250 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb +2025-01-06 22:34:29,454 DEBUG SenderThread:2127250 [sender.py:send():379] send: header +2025-01-06 22:34:29,463 DEBUG SenderThread:2127250 [sender.py:send():379] send: run +2025-01-06 22:34:29,700 INFO SenderThread:2127250 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files +2025-01-06 22:34:29,700 INFO SenderThread:2127250 [sender.py:_start_run_threads():1188] run started: hxim6a4m with start time 1736220869.441899 +2025-01-06 22:34:29,715 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:34:29,716 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:34:29,763 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:34:29,767 DEBUG HandlerThread:2127250 [system_info.py:__init__():26] System info init +2025-01-06 22:34:29,767 DEBUG HandlerThread:2127250 [system_info.py:__init__():41] System info init done +2025-01-06 22:34:29,767 INFO HandlerThread:2127250 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:34:29,768 INFO SystemMonitor:2127250 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:34:29,768 INFO HandlerThread:2127250 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:34:29,768 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:34:29,769 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:34:29,769 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:34:29,770 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:34:29,771 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:34:29,780 DEBUG HandlerThread:2127250 [system_info.py:probe():152] Probing system +2025-01-06 22:34:29,782 DEBUG HandlerThread:2127250 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:probe():200] Probing system done +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:34:29.780802', 'startedAt': '2025-01-07T03:34:29.433275', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.900125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.2, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 590.0657348632812}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:34:29,787 INFO HandlerThread:2127250 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:34:29,787 INFO HandlerThread:2127250 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:_save_code():44] Saving code +2025-01-06 22:34:29,790 DEBUG HandlerThread:2127250 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:34:29,790 DEBUG HandlerThread:2127250 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:34:29,853 DEBUG HandlerThread:2127250 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:34:29,854 INFO HandlerThread:2127250 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:34:29,857 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:34:30,096 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:34:30,096 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:34:30,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:30,099 DEBUG SenderThread:2127250 [sender.py:send():379] send: telemetry +2025-01-06 22:34:30,114 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/k5hddnfx-wandb-metadata.json +2025-01-06 22:34:30,131 INFO wandb-upload_1:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/2got678c-code/FISH/eval_robot.py +2025-01-06 22:34:30,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:34:30,145 INFO HandlerThread:2127250 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/tb" + save: true +} +control { + relay_id: "d6c038ba558f4ae089ac31ed308dcd2b" +} +_info { + stream_id: "hxim6a4m" +} + +2025-01-06 22:34:30,145 DEBUG HandlerThread:2127250 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:34:30,146 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:34:30,155 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/7ih0tl8e-diff.patch +2025-01-06 22:34:30,275 DEBUG SenderThread:2127250 [sender.py:send():379] send: tbrecord +2025-01-06 22:34:30,275 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:34:30,275 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 with policy live +2025-01-06 22:34:30,462 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/606yvhpn-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:34:30,701 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH +2025-01-06 22:34:31,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:32,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:32,701 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:33,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:34,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:35,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:35,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:35,686 DEBUG SenderThread:2127250 [sender.py:send():379] send: config +2025-01-06 22:34:36,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:36,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:37,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:38,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:38,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:39,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:40,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:40,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:40,703 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:41,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:42,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:42,704 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:43,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:44,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:44,704 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:45,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:34:45,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:34:45,137 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:45,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:46,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:46,705 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:47,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:48,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:48,705 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:49,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:50,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:50,706 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:51,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:51,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:52,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:52,707 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:53,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:54,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:55,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:56,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:57,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:57,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:58,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:59,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:00,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:00,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:01,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:02,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:02,180 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:02,709 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:35:03,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:04,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:05,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:06,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:07,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:08,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:08,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:09,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:10,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:11,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:12,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:13,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:13,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:14,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:15,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:15,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:15,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:16,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:17,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:18,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:18,237 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:19,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:20,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:21,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:22,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:23,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:23,899 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:24,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:24,894 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:35:24,894 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 with policy now +2025-01-06 22:35:24,894 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:35:24,895 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:35:24,895 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:35:24,896 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:35:25,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:25,160 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/w7y83qoe-media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:25,715 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos +2025-01-06 22:35:25,985 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/8jyx1oys-media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:26,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:26,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:27,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:28,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:29,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:29,553 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:29,771 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:35:29,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:35:30,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:30,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:30,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:30,717 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:31,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:32,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:32,718 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:33,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:34,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:35,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:35,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:36,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:37,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:38,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:39,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:40,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:40,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:41,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:41,721 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:35:41,961 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/v5os8x6w-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:35:42,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:42,721 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:43,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:44,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:44,722 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:45,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:45,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:45,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:45,212 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:46,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:46,723 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:47,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:48,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:49,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:50,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:51,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:51,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:52,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:53,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:54,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:55,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:56,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:56,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:57,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:58,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:59,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:59,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:36:00,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:00,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:01,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:01,157 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:02,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:03,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:04,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:05,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:06,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:07,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:07,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:08,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:09,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:10,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:11,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:12,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:12,731 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:12,809 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:13,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:14,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:15,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:15,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:15,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:16,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:17,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:18,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:18,254 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:19,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:20,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:21,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:21,500 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:36:21,500 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 with policy now +2025-01-06 22:36:21,507 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:36:21,507 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:36:21,507 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:36:21,514 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:36:21,727 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/f6jm88u7-media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:36:22,010 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/xiz6ndb8-media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:22,116 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:22,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:23,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:23,526 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:23,735 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:24,115 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:25,116 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:26,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:26,736 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:27,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:28,117 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:28,737 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:28,897 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:29,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:29,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:36:30,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:30,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:30,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:31,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:32,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:33,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:33,738 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:36:34,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:34,525 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:34,579 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:36:34,579 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:36:34,579 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:36:34,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:36:34,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:35,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:36,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:36,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:37,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:38,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:38,740 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:39,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:40,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:40,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:41,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:42,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:43,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:44,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:45,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:45,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:45,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:45,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:46,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:47,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:48,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:49,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:50,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:51,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:51,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:52,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:53,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:54,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:55,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:56,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:56,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:57,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:58,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:59,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:59,773 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:37:00,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:00,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:01,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:01,360 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:02,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:02,748 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:03,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:04,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:05,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:06,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:06,750 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:07,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:07,309 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:08,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:09,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:10,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:11,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:12,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:12,805 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:13,143 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:13,163 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:37:13,163 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 with policy now +2025-01-06 22:37:13,205 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:37:13,206 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:37:13,206 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:37:13,212 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:37:13,393 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/m3h2jkam-media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:37:14,011 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/u3ybg8g6-media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:14,135 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:14,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:15,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:15,106 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:15,149 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:16,134 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:17,137 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:17,849 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:18,136 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:18,754 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:19,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:20,135 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:20,754 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:21,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:22,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:23,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:23,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:24,143 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:25,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:26,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:27,142 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:28,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:28,142 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:29,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:29,773 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:37:30,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:30,104 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:30,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:31,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:32,146 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:33,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:33,253 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:34,146 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:35,148 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:36,151 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:37,150 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:38,148 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:39,114 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:39,150 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:40,154 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:41,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:42,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:43,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:44,155 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:45,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:45,105 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:45,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:45,209 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:46,155 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:47,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:48,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:49,159 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:50,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:51,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:51,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:52,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:53,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:54,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:55,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:56,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:56,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:57,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:58,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:58,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:37:59,013 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/49678ylt-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:37:59,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:59,447 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:37:59,447 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:37:59,448 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:37:59,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:37:59,774 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:38:00,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:00,113 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:00,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:00,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:01,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:01,239 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:02,170 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:03,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:04,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:04,769 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:05,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:06,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:07,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:07,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:08,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:09,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:10,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:11,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:12,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:12,780 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:13,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:14,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:14,772 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:15,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:15,102 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:15,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:16,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:17,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:18,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:18,275 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:19,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:20,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:21,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:22,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:23,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:24,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:24,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:25,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:26,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:27,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:28,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:29,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:29,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:29,774 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:38:30,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:30,102 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:30,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:31,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:32,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:33,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:34,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:34,271 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:35,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:36,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:37,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:38,171 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:38,590 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:38:38,590 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 with policy now +2025-01-06 22:38:38,599 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:38:38,600 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:38:38,600 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:38:38,622 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:38:38,822 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/jq186otc-media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:39,102 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/gnugbyb0-media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:39,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:39,633 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:40,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:40,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:41,188 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:42,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:43,173 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:44,172 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:44,782 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:45,041 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:45,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:45,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:45,171 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:46,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:46,782 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:47,178 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:48,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:49,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:50,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:50,180 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:50,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:38:51,178 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:51,507 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:38:51,507 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:38:51,509 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:38:51,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:38:52,181 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:52,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:53,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:54,187 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:55,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:55,190 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:56,182 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:56,786 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:57,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:58,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:59,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:59,775 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:00,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:00,104 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:00,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:00,278 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:01,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:02,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:03,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:04,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:04,789 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:05,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:06,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:06,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:07,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:08,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:09,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:10,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:11,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:11,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:12,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:13,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:14,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:15,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:15,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:15,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:16,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:16,213 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:17,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:18,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:18,793 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:19,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:20,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:21,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:22,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:22,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:23,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:24,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:24,795 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:25,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:26,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:27,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:27,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:28,187 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:29,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:29,775 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:30,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:30,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:30,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:31,200 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:31,379 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:31,379 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 with policy now +2025-01-06 22:39:31,380 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:31,380 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:31,380 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:31,402 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:31,609 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/h54swvyd-media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:39:32,061 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/pyh23mby-media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:32,203 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:32,420 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:32,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:33,204 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:34,202 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:35,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:36,199 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:36,799 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:37,209 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:37,831 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:38,208 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:38,800 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:39,211 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:40,211 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:40,550 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:39:40,550 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:39:40,763 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media +2025-01-06 22:39:40,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval +2025-01-06 22:39:40,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table +2025-01-06 22:39:41,027 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/3sa2ksj1-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:41,049 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/zuxk2h5d-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:41,089 INFO wandb-upload_1:2127250 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjbixg0zu +2025-01-06 22:39:41,126 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:39:41,139 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:41,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:41,458 INFO SenderThread:2127250 [sender.py:send_request_log_artifact():1525] logged artifact run-hxim6a4m-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYxMzM1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc1Mjk3MQ==', 'latestArtifact': None}} +2025-01-06 22:39:41,458 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:41,459 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,459 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,459 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:41,459 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,460 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send():379] send: telemetry +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,461 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,461 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,462 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,463 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,463 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,463 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,464 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,464 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,464 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,464 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,465 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,465 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,466 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,466 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,467 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,467 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:39:41,708 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/1z929dbw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval +2025-01-06 22:39:41,867 INFO wandb-upload_1:2127250 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_6jrosws +2025-01-06 22:39:42,001 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/423iz59p-media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:42,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:42,259 INFO SenderThread:2127250 [sender.py:send_request_log_artifact():1525] logged artifact run-hxim6a4m-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYxMzM3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc1Mjk3OQ==', 'latestArtifact': None}} +2025-01-06 22:39:42,259 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:42,259 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json with policy now +2025-01-06 22:39:42,260 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:42,260 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:42,261 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:42,508 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/k2ue8032-media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:42,621 DEBUG SenderThread:2127250 [sender.py:send():379] send: exit +2025-01-06 22:39:42,621 INFO SenderThread:2127250 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_exit():588] handling runtime: 312 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_exit():594] send defer +2025-01-06 22:39:42,622 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,622 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:39:42,622 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:39:42,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:39:42,623 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,623 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:39:42,623 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:39:42,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:39:42,623 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:39:42,623 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:39:42,624 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:39:42,624 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:39:42,675 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:39:42,676 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:39:42,676 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:39:42,676 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,676 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:39:42,676 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:39:42,676 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:42,677 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,677 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:39:42,677 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,677 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:39:42,677 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:39:42,677 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,677 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:39:42,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:42,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:44,803 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,570 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,570 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:39:49,570 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:39:49,570 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:49,571 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:49,571 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:49,572 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,572 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:39:49,572 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:49,573 DEBUG SenderThread:2127250 [sender.py:send():379] send: summary +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:49,573 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:39:49,573 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,574 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:39:49,574 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,574 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:39:49,577 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:49,724 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:39:49,724 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:49,725 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,725 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:39:49,725 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,725 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:39:49,805 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:49,805 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:39:50,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:51,605 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:39:51,605 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:51,606 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:51,606 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:39:51,606 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:51,606 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:39:51,606 INFO SenderThread:2127250 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:39:51,608 INFO SenderThread:2127250 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:39:51,610 INFO SenderThread:2127250 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:39:51,621 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:39:51,621 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:51,621 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:39:51,622 DEBUG SenderThread:2127250 [sender.py:send():379] send: artifact +2025-01-06 22:39:51,624 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:51,806 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'versionIndex': 428}}} +2025-01-06 22:39:51,827 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml config.yaml +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json wandb-summary.json +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt requirements.txt +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log output.log +2025-01-06 22:39:52,811 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch diff.patch +2025-01-06 22:39:52,813 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:39:52,816 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:52,816 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:52,816 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:39:52,817 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:52,817 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:39:52,817 INFO SenderThread:2127250 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:39:52,990 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt +2025-01-06 22:39:52,994 INFO wandb-upload_1:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:53,007 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:39:53,044 INFO wandb-upload_4:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:53,117 INFO wandb-upload_3:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/er0j79x4-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:53,317 INFO Thread-11 (_thread_body):2127250 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:39:53,317 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,317 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:39:53,317 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,317 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:39:53,318 INFO SenderThread:2127250 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:39:53,319 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,319 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:39:53,319 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:39:53,625 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,688 INFO SenderThread:2127250 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:39:53,688 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:39:53,689 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,689 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,689 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:39:53,689 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,689 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:39:53,689 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:39:53,689 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,689 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send():379] send: final +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send():379] send: footer +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,690 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:39:53,690 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:39:53,691 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,691 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:39:53,693 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:39:53,694 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:53,755 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,755 INFO MainThread:2127250 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:39:53,756 INFO MainThread:2127250 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:39:53,756 INFO MainThread:2127250 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:39:53,756 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:39:53,756 INFO HandlerThread:2127250 [handler.py:finish():882] shutting down handler +2025-01-06 22:39:54,691 INFO WriterThread:2127250 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [sender.py:finish():1615] shutting down sender +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [file_pusher.py:join():175] waiting for file pusher diff --git a/hxim6a4m/wandb/debug.log b/hxim6a4m/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ae1330f46f227b35b6bf386e85f71b63e0ca2ddf --- /dev/null +++ b/hxim6a4m/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:34:29,434 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Configure stats pid to 2127077 +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/settings +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug.log +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug-internal.log +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m'} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():619] starting backend +2025-01-06 22:34:29,436 INFO MainThread:2127077 [wandb_init.py:init():623] setting up manager +2025-01-06 22:34:29,440 INFO MainThread:2127077 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:34:29,441 INFO MainThread:2127077 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:34:29,453 INFO MainThread:2127077 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:34:29,461 INFO MainThread:2127077 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:34:29,715 INFO MainThread:2127077 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:34:29,757 INFO MainThread:2127077 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:34:29,757 INFO MainThread:2127077 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:34:30,098 INFO MainThread:2127077 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:34:30,100 INFO MainThread:2127077 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:34:30,101 INFO MainThread:2127077 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/tb, True +2025-01-06 22:34:35,685 INFO MainThread:2127077 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:39:54,757 WARNING MsgRouterThr:2127077 [router.py:message_loop():77] message_loop has been closed diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4c59171b87c31300905aefddf5d04ba3d0f7724 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1003_0' +wandb_notes: + desc: null + value: 1003_0_req_1364_0restarted_3 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736220869 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c03766eedde4e00adf1f118f93a1d1dd2d31994e --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..1643b06 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a75d0b11fd6c46eb0b5fe8dccfcca1dd7420d872 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[30, 30, 1.0, 0.8705282806609103, 1, 0.9202661215418471, 1, 0.9387300535159031, 1, 0.9334393589821826, 1]]} \ No newline at end of file diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0a49b792e8763d0cf3dc671f4b28b0e97f03757 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ff36b24aad6fe917e51fa7450a454bb835a8553386e2cdcfd25661ee99a59e +size 1414528 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..59db9d0af910e01e8eb2fdb534a1ca665b3fb861 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51fb3af7bcef5065307e5065de0b4e80b112e4bfb5e31eab835d14079a4b9e55 +size 1460297 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d12d876a86d2ea59031e5dd6deb872d84a4b4a96 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3f7d823766e09778f611008af23da5576adbf3900f797a867398b10a45c0ff +size 1498734 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8169bf2f89034830d966a5bdd81811f566c73e32 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f961b9a58ac3c44eab41a31a18131c71e967e8208a35af5b482ced5f81d5b87a +size 1430092 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbc6c5300e6556f4e6f531ba1e1ca1a05416f791 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5fec9c26b35ac2c23952b6d8f2b84c13f5bb9670b89849d5d6203bcb34283e +size 1453160 diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4cb4e46055fa11d55669307c373ab74917b184eb --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736220875.686844]: resetting environment +[INFO] [1736220875.694957]: cleared current plan +[INFO] [1736220875.695964]: moving to home +[INFO] [1736220876.697309]: reached home +[INFO] [1736220876.698689]: reset action history +[INFO] [1736220879.408643]: environment reset +Starting episode 0 +[INFO] [1736220879.413010]: resetting environment +[INFO] [1736220879.420827]: cleared current plan +[INFO] [1736220879.421970]: moving to home +[INFO] [1736220880.424361]: reached home +[INFO] [1736220880.426064]: reset action history +[INFO] [1736220883.135728]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736220885.365736]: resetting environment +[INFO] [1736220885.369949]: cleared current plan +[INFO] [1736220885.370802]: moving to home +[INFO] [1736220886.373645]: reached home +[INFO] [1736220886.375979]: reset action history +[INFO] [1736220889.135062]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/episode_rosbags/episode_0_2025-01-06-22-34-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2127077.log +started roslaunch server http://158.130.50.37:33601/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2127361] +process[print_text-2]: started with pid [2127362] +process[rosbag_record-3]: started with pid [2127386] +started bagging! +For topic gripper_width: timestamp difference is 102296056 for nearest: 1736220891757338466 - target: 1736220891655042410 at idx 37 +[INFO] [1736220923.898695]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736220924.690093]: Stored episode 1. +[INFO] [1736220924.690331]: Saving video... +[INFO] [1736220924.882900]: Video saved! +Starting episode 1 +[INFO] [1736220924.895073]: resetting environment +[INFO] [1736220924.906743]: cleared current plan +[INFO] [1736220924.906975]: moving to home +[INFO] [1736220928.509316]: reached home +[INFO] [1736220928.510317]: reset action history +[INFO] [1736220931.221813]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220941.593598]: resetting environment +[INFO] [1736220941.598893]: cleared current plan +[INFO] [1736220941.599567]: moving to home +[INFO] [1736220942.601304]: reached home +[INFO] [1736220942.601764]: reset action history +[INFO] [1736220945.309596]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/episode_rosbags/episode_1_2025-01-06-22-35-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2127077.log +started roslaunch server http://158.130.50.37:36033/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2127510] +process[print_text-5]: started with pid [2127511] +process[rosbag_record-6]: started with pid [2127535] +started bagging! +For topic gripper_width: timestamp difference is 100231428 for nearest: 1736220971757389803 - target: 1736220971657158375 at idx 249 +[INFO] [1736220980.058607]: Storing episode... +For topic gripper_width: timestamp difference is 106944231 for nearest: 1736220980090695051 - target: 1736220979983750820 at idx 249 +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736220980.848204]: Stored episode 2. +[INFO] [1736220980.848510]: Saving video... +[INFO] [1736220981.470288]: Video saved! +Starting episode 2 +[INFO] [1736220981.517057]: resetting environment +[INFO] [1736220981.524966]: cleared current plan +[INFO] [1736220981.525207]: moving to home +[WARN] [1736220981.411137]: Plan exhausted +[WARN] [1736220981.451996]: Plan exhausted +[WARN] [1736220981.491144]: Plan exhausted +[INFO] [1736220985.128232]: reached home +[INFO] [1736220985.129371]: reset action history +[INFO] [1736220987.853576]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220993.511987]: resetting environment +[INFO] [1736220993.520265]: cleared current plan +[INFO] [1736220993.520505]: moving to home +[INFO] [1736220994.521521]: reached home +[INFO] [1736220994.522533]: reset action history +[INFO] [1736220997.236499]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/episode_rosbags/episode_2_2025-01-06-22-36-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2127077.log +started roslaunch server http://158.130.50.37:33557/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2127655] +process[print_text-8]: started with pid [2127656] +process[rosbag_record-9]: started with pid [2127680] +started bagging! +For topic gripper_width: timestamp difference is 106183173 for nearest: 1736221020557388188 - target: 1736221020451205015 at idx 249 +For topic gripper_width: timestamp difference is 107793115 for nearest: 1736221021357353948 - target: 1736221021249560833 at idx 249 +For topic gripper_width: timestamp difference is 116004541 for nearest: 1736221025257414177 - target: 1736221025141409636 at idx 249 +[INFO] [1736221032.063056]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736221032.802590]: Stored episode 3. +[INFO] [1736221032.805721]: Saving video... +[INFO] [1736221033.118600]: Video saved! +Starting episode 3 +[INFO] [1736221033.179338]: resetting environment +[INFO] [1736221033.200471]: cleared current plan +[INFO] [1736221033.200788]: moving to home +[INFO] [1736221036.805206]: reached home +[INFO] [1736221036.808372]: reset action history +[INFO] [1736221039.686222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221078.755658]: resetting environment +[INFO] [1736221078.775560]: cleared current plan +[INFO] [1736221078.775783]: moving to home +[INFO] [1736221079.777251]: reached home +[INFO] [1736221079.780930]: reset action history +[INFO] [1736221082.506430]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/episode_rosbags/episode_3_2025-01-06-22-38-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2127077.log +started roslaunch server http://158.130.50.37:41203/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2127860] +process[print_text-11]: started with pid [2127861] +process[rosbag_record-12]: started with pid [2127862] +started bagging! +For topic gripper_width: timestamp difference is 100420810 for nearest: 1736221092757427789 - target: 1736221092657006979 at idx 139 +[INFO] [1736221117.489315]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1736221118.270320]: Stored episode 4. +[INFO] [1736221118.285169]: Saving video... +[INFO] [1736221118.525652]: Video saved! +Starting episode 4 +[INFO] [1736221118.610772]: resetting environment +[INFO] [1736221118.632668]: cleared current plan +[INFO] [1736221118.632935]: moving to home +[INFO] [1736221122.235943]: reached home +[INFO] [1736221122.238048]: reset action history +[INFO] [1736221125.040680]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736221130.724046]: resetting environment +[INFO] [1736221130.747918]: cleared current plan +[INFO] [1736221130.754001]: moving to home +[INFO] [1736221131.758021]: reached home +[INFO] [1736221131.764973]: reset action history +[INFO] [1736221134.491343]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/episode_rosbags/episode_4_2025-01-06-22-38-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2127077.log +started roslaunch server http://158.130.50.37:35657/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2128005] +process[print_text-14]: started with pid [2128006] +process[rosbag_record-15]: started with pid [2128007] +started bagging! +For topic gripper_width: timestamp difference is 105001088 for nearest: 1736221142657387610 - target: 1736221142552386522 at idx 110 +For topic gripper_width: timestamp difference is 117236063 for nearest: 1736221157790720388 - target: 1736221157673484325 at idx 249 +For topic EE_pose: timestamp difference is 129646906 for nearest: 1736221163403601536 - target: 1736221163273954630 at idx 0 +For topic EE_pose: timestamp difference is 280732211 for nearest: 1736221163554686841 - target: 1736221163273954630 at idx 0 +[INFO] [1736221170.296011]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736221171.016357]: Stored episode 5. +[INFO] [1736221171.021947]: Saving video... +[INFO] [1736221171.326072]: Video saved! +[INFO] [1736221171.382890]: resetting environment +[INFO] [1736221171.416771]: cleared current plan +[INFO] [1736221171.419498]: moving to home +[INFO] [1736221175.026424]: reached home +[INFO] [1736221175.043662]: reset action history +[INFO] [1736221177.828007]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..15aeda7a133ac6e239164e221db993681221cd69 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:34:29.780802", + "startedAt": "2025-01-07T03:34:29.433275", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.900125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.2, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.005, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.938, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 590.0657348632812 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b3ee2da9c74b164a4c9ca8d78066d554084c90d6 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c0ff36b24aad6fe917e51fa7450a454bb835a8553386e2cdcfd25661ee99a59e", "size": 1414528, "path": "media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4"}, "global_step": 665, "_timestamp": 1736221181.047398, "_runtime": 311.60549902915955, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "51fb3af7bcef5065307e5065de0b4e80b112e4bfb5e31eab835d14079a4b9e55", "size": 1460297, "path": "media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4"}, "eval/num_success": 5.0, "episode": 30.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "2a3f7d823766e09778f611008af23da5576adbf3900f797a867398b10a45c0ff", "size": 1498734, "path": "media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f961b9a58ac3c44eab41a31a18131c71e967e8208a35af5b482ced5f81d5b87a", "size": 1430092, "path": "media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4a5fec9c26b35ac2c23952b6d8f2b84c13f5bb9670b89849d5d6203bcb34283e", "size": 1453160, "path": "media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://pl7efljfnlclcp8nhssxcjjat4r1m8a9x7ru6lbuyutbcifeeqssg0m2c7ljyybu98gvilfnwioh3yol5uhrxnkb3v4qxa5r1crph6tbbt39yvdgtb3ppp6okl8guksk/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://lnf1hhjvhnks2bqjw0dx9syshrvyv76381xada68afdp9dshh95g58dl8s3kf9fi729z81en3n9dunzzabazf6r8er5bgxmzni256feqwpraxtpfd728dh9aqq26gxy6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 30, "total_num_episodes": 30, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.8705282806609103, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9202661215418471, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9387300535159031, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.9334393589821826, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "70e918bc0a9fb473bc5b24faf10a8ecae8ea4cab85d3bc2714117749dad4921b", "size": 437, "artifact_path": "wandb-client-artifact://gvjmoxjjm6ikgf6j060y46nwlq3wsjqiz2mvgghsrhq7lbkrlb3jz6viukjc6e9ku6g3uyxk9smp0ir8rris9jlzh3rld8olivoxjledr1vfyusr2brnp019b8f1odar/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zea10mmbc1j5n9vtx9vii082ytqyky527lpg2bli3sdhwkqdgjsz4b728viu2tjndfy15p2gthqsaan2og4w539qfmbjl9fn8eaeaf0izz3hes83xk7qfh97il1km0lh:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 312}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.8705282807350159, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9202660918235779, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9387300610542297, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.9334393739700317, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug-internal.log b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..484a530a02b853878a7131f9254e0466da5640d6 --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug-internal.log @@ -0,0 +1,833 @@ +2025-01-06 22:34:29,441 INFO StreamThr :2127250 [internal.py:wandb_internal():85] W&B internal server running at pid: 2127250, started at: 2025-01-06 22:34:29.440912 +2025-01-06 22:34:29,442 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:34:29,453 INFO WriterThread:2127250 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb +2025-01-06 22:34:29,454 DEBUG SenderThread:2127250 [sender.py:send():379] send: header +2025-01-06 22:34:29,463 DEBUG SenderThread:2127250 [sender.py:send():379] send: run +2025-01-06 22:34:29,700 INFO SenderThread:2127250 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files +2025-01-06 22:34:29,700 INFO SenderThread:2127250 [sender.py:_start_run_threads():1188] run started: hxim6a4m with start time 1736220869.441899 +2025-01-06 22:34:29,715 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:34:29,716 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:34:29,763 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:34:29,767 DEBUG HandlerThread:2127250 [system_info.py:__init__():26] System info init +2025-01-06 22:34:29,767 DEBUG HandlerThread:2127250 [system_info.py:__init__():41] System info init done +2025-01-06 22:34:29,767 INFO HandlerThread:2127250 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:34:29,768 INFO SystemMonitor:2127250 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:34:29,768 INFO HandlerThread:2127250 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:34:29,768 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:34:29,769 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:34:29,769 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:34:29,770 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:34:29,771 INFO SystemMonitor:2127250 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:34:29,780 DEBUG HandlerThread:2127250 [system_info.py:probe():152] Probing system +2025-01-06 22:34:29,782 DEBUG HandlerThread:2127250 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:probe():200] Probing system done +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:34:29.780802', 'startedAt': '2025-01-07T03:34:29.433275', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.900125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.2, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.005, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 590.0657348632812}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:34:29,787 INFO HandlerThread:2127250 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:34:29,787 INFO HandlerThread:2127250 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:34:29,787 DEBUG HandlerThread:2127250 [system_info.py:_save_code():44] Saving code +2025-01-06 22:34:29,790 DEBUG HandlerThread:2127250 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:34:29,790 DEBUG HandlerThread:2127250 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:34:29,853 DEBUG HandlerThread:2127250 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:34:29,854 INFO HandlerThread:2127250 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:34:29,857 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:34:29,857 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:34:30,096 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:34:30,096 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:34:30,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:30,099 DEBUG SenderThread:2127250 [sender.py:send():379] send: telemetry +2025-01-06 22:34:30,114 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/k5hddnfx-wandb-metadata.json +2025-01-06 22:34:30,131 INFO wandb-upload_1:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/2got678c-code/FISH/eval_robot.py +2025-01-06 22:34:30,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:34:30,145 INFO HandlerThread:2127250 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/tb" + save: true +} +control { + relay_id: "d6c038ba558f4ae089ac31ed308dcd2b" +} +_info { + stream_id: "hxim6a4m" +} + +2025-01-06 22:34:30,145 DEBUG HandlerThread:2127250 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:34:30,146 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:34:30,155 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/7ih0tl8e-diff.patch +2025-01-06 22:34:30,275 DEBUG SenderThread:2127250 [sender.py:send():379] send: tbrecord +2025-01-06 22:34:30,275 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:34:30,275 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 with policy live +2025-01-06 22:34:30,462 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/606yvhpn-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:34:30,701 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code +2025-01-06 22:34:30,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH +2025-01-06 22:34:31,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:32,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:32,701 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:33,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:34,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:35,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:35,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:35,686 DEBUG SenderThread:2127250 [sender.py:send():379] send: config +2025-01-06 22:34:36,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:36,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:37,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:38,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:38,702 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:39,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:40,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:40,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:40,703 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:41,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:42,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:42,704 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:43,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:44,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:44,704 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:45,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:34:45,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:34:45,137 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:45,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:46,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:46,705 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:47,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:48,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:48,705 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:49,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:50,099 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:50,706 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:51,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:51,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:52,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:52,707 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:34:53,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:54,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:55,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:56,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:57,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:34:57,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:58,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:34:59,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:00,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:00,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:01,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:02,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:02,180 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:02,709 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:35:03,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:04,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:05,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:06,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:07,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:08,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:08,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:09,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:10,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:11,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:12,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:13,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:13,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:14,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:15,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:15,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:15,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:16,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:17,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:18,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:18,237 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:19,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:20,104 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:21,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:22,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:23,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:23,899 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:24,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:24,894 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:35:24,894 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 with policy now +2025-01-06 22:35:24,894 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:35:24,895 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:35:24,895 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:35:24,896 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:35:25,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:25,160 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/w7y83qoe-media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:25,715 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:35:25,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos +2025-01-06 22:35:25,985 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/8jyx1oys-media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:35:26,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:26,716 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:27,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:28,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:29,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:29,553 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:29,771 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:35:29,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:35:30,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:30,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:30,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:30,717 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:31,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:32,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:32,718 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:33,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:34,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:35,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:35,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:36,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:37,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:38,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:39,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:40,100 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:40,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:41,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:41,721 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:35:41,961 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/v5os8x6w-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:35:42,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:42,721 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:43,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:44,107 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:44,722 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:45,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:35:45,097 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:35:45,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:45,212 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:46,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:46,723 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:35:47,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:48,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:49,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:50,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:51,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:51,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:52,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:53,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:54,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:55,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:56,101 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:35:56,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:57,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:58,109 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:59,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:35:59,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:36:00,097 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:00,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:01,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:01,157 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:02,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:03,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:04,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:05,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:06,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:07,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:07,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:08,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:09,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:10,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:11,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:12,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:12,731 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:12,809 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:13,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:14,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:15,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:15,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:15,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:16,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:17,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:18,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:18,254 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:19,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:20,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:21,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:21,500 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:36:21,500 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 with policy now +2025-01-06 22:36:21,507 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:36:21,507 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:36:21,507 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:36:21,514 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:36:21,727 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/f6jm88u7-media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:21,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:36:22,010 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/xiz6ndb8-media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:36:22,116 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:22,734 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:23,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:23,526 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:23,735 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:24,115 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:25,116 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:26,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:26,736 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:27,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:28,117 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:28,737 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:28,897 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:29,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:29,772 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:36:30,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:30,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:30,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:31,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:32,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:33,118 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:33,738 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:36:34,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:34,525 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:34,579 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:36:34,579 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:36:34,579 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:36:34,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:36:34,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:35,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:36,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:36,739 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:37,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:38,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:38,740 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:36:39,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:40,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:40,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:41,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:42,119 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:43,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:44,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:45,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:36:45,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:36:45,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:45,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:46,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:47,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:48,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:49,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:50,120 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:51,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:51,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:52,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:53,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:54,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:55,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:56,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:36:56,121 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:57,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:58,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:59,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:36:59,773 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:37:00,098 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:00,098 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:00,141 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:01,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:01,360 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:02,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:02,748 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:03,122 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:04,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:05,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:06,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:06,750 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:07,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:07,309 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:08,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:09,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:10,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:11,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:12,123 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:12,805 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:13,143 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:13,163 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:37:13,163 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 with policy now +2025-01-06 22:37:13,205 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:37:13,206 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:37:13,206 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:37:13,212 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:37:13,393 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/m3h2jkam-media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:13,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:37:14,011 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/u3ybg8g6-media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:37:14,135 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:14,752 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:15,106 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:15,106 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:15,149 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:16,134 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:17,137 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:17,849 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:18,136 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:18,754 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:19,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:20,135 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:20,754 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:37:21,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:22,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:23,108 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:23,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:24,143 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:25,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:26,140 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:27,142 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:28,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:28,142 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:29,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:29,773 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:37:30,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:30,104 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:30,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:31,144 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:32,146 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:33,145 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:33,253 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:34,146 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:35,148 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:36,151 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:37,150 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:38,148 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:39,114 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:39,150 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:40,154 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:41,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:42,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:43,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:44,155 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:45,105 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:37:45,105 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:37:45,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:45,209 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:46,155 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:47,156 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:48,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:49,159 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:50,158 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:51,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:51,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:52,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:53,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:54,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:55,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:56,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:37:56,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:57,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:58,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:58,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:37:59,013 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/49678ylt-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:37:59,161 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:37:59,447 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:37:59,447 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:37:59,448 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:37:59,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:37:59,774 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:38:00,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:00,113 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:00,163 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:00,767 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:01,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:01,239 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:02,170 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:03,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:04,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:04,769 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:05,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:06,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:07,110 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:07,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:08,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:09,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:10,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:11,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:12,164 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:12,780 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:13,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:14,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:14,772 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:15,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:15,102 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:15,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:16,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:17,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:18,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:18,275 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:19,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:20,165 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:21,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:22,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:23,166 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:24,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:24,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:25,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:26,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:27,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:28,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:29,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:29,167 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:29,774 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:38:30,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:30,102 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:30,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:31,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:32,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:33,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:34,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:34,271 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:35,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:36,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:37,168 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:38,171 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:38,590 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:38:38,590 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 with policy now +2025-01-06 22:38:38,599 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:38:38,600 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:38:38,600 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:38:38,622 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:38,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:38:38,822 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/jq186otc-media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:39,102 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/gnugbyb0-media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:38:39,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:39,633 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:40,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:40,780 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:41,188 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:42,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:43,173 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:44,172 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:44,782 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:45,041 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:45,102 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:38:45,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:38:45,171 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:46,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:46,782 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:47,178 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:48,179 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:49,177 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:50,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:50,180 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:50,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:38:51,178 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:51,507 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:38:51,507 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:38:51,509 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:38:51,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:38:52,181 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:52,784 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:53,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:54,187 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:55,113 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:38:55,190 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:56,182 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:56,786 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:38:57,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:58,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:59,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:38:59,775 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:00,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:00,104 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:00,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:00,278 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:01,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:02,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:03,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:04,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:04,789 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:05,183 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:06,111 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:06,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:07,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:08,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:09,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:10,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:11,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:11,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:12,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:13,184 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:14,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:15,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:15,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:15,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:16,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:16,213 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:17,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:18,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:18,793 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:19,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:20,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:21,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:22,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:22,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:23,185 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:24,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:24,795 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:25,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:26,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:27,112 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:27,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:28,187 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:29,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:29,775 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:30,103 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:39:30,103 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:39:30,186 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:31,200 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:31,379 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:31,379 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 with policy now +2025-01-06 22:39:31,380 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:31,380 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:31,380 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:31,402 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:31,609 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/h54swvyd-media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:31,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval +2025-01-06 22:39:32,061 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/pyh23mby-media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:32,203 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:32,420 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:32,798 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:33,204 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:34,202 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:35,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:36,199 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:36,799 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:37,209 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:37,831 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:38,208 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:38,800 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:39,211 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:40,211 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:40,550 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:39:40,550 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:39:40,763 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:40,801 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media +2025-01-06 22:39:40,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval +2025-01-06 22:39:40,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table +2025-01-06 22:39:41,027 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/3sa2ksj1-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:41,049 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/zuxk2h5d-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:41,089 INFO wandb-upload_1:2127250 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjbixg0zu +2025-01-06 22:39:41,126 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:39:41,139 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:39:41,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:41,458 INFO SenderThread:2127250 [sender.py:send_request_log_artifact():1525] logged artifact run-hxim6a4m-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYxMzM1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc1Mjk3MQ==', 'latestArtifact': None}} +2025-01-06 22:39:41,458 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:41,459 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,459 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,459 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:41,459 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,460 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send():379] send: telemetry +2025-01-06 22:39:41,460 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,461 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,461 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,462 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,463 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,463 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,463 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,464 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,464 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,464 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,464 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,465 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,465 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,465 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,466 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,466 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,466 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:41,467 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:41,467 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:39:41,708 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/1z929dbw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:41,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval +2025-01-06 22:39:41,867 INFO wandb-upload_1:2127250 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_6jrosws +2025-01-06 22:39:42,001 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/423iz59p-media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:42,206 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:42,259 INFO SenderThread:2127250 [sender.py:send_request_log_artifact():1525] logged artifact run-hxim6a4m-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjYxMzM3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc1Mjk3OQ==', 'latestArtifact': None}} +2025-01-06 22:39:42,259 DEBUG SenderThread:2127250 [sender.py:send():379] send: files +2025-01-06 22:39:42,259 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json with policy now +2025-01-06 22:39:42,260 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:42,260 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:42,261 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:42,508 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/k2ue8032-media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:42,621 DEBUG SenderThread:2127250 [sender.py:send():379] send: exit +2025-01-06 22:39:42,621 INFO SenderThread:2127250 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_exit():588] handling runtime: 312 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_exit():594] send defer +2025-01-06 22:39:42,622 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,622 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:39:42,622 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:39:42,622 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:39:42,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:39:42,623 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,623 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:39:42,623 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:39:42,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:39:42,623 INFO HandlerThread:2127250 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:39:42,623 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:39:42,623 DEBUG SystemMonitor:2127250 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:39:42,624 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:39:42,624 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:39:42,675 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:39:42,676 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:39:42,676 INFO HandlerThread:2127250 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:39:42,676 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,676 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:39:42,676 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:39:42,676 DEBUG SenderThread:2127250 [sender.py:send():379] send: stats +2025-01-06 22:39:42,677 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,677 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:39:42,677 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:42,677 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:39:42,677 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:39:42,677 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:42,677 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:39:42,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:42,802 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:44,803 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:39:49,569 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,570 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,570 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:39:49,570 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:39:49,570 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:49,571 DEBUG SenderThread:2127250 [sender.py:send():379] send: history +2025-01-06 22:39:49,571 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:39:49,572 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,572 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:39:49,572 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:49,573 DEBUG SenderThread:2127250 [sender.py:send():379] send: summary +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:39:49,573 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:39:49,573 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:39:49,573 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,574 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:39:49,574 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,574 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:39:49,577 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:39:49,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:49,724 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:39:49,724 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:49,725 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:49,725 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:39:49,725 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:49,725 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:39:49,805 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:49,805 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:39:50,623 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:51,605 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:39:51,605 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:51,606 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:51,606 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:39:51,606 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:51,606 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:39:51,606 INFO SenderThread:2127250 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:39:51,608 INFO SenderThread:2127250 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:39:51,610 INFO SenderThread:2127250 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:39:51,621 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:39:51,621 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:51,621 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:39:51,622 DEBUG SenderThread:2127250 [sender.py:send():379] send: artifact +2025-01-06 22:39:51,624 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:51,806 INFO Thread-12 :2127250 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMjU5OTQ4NA==', 'versionIndex': 428}}} +2025-01-06 22:39:51,827 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:39:51,827 INFO SenderThread:2127250 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml config.yaml +2025-01-06 22:39:52,807 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json wandb-summary.json +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt requirements.txt +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:52,809 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log output.log +2025-01-06 22:39:52,811 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/diff.patch diff.patch +2025-01-06 22:39:52,813 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 media/videos/eval/0_eval_0_c0ff36b24aad6fe917e5.mp4 +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 media/videos/eval/2_eval_2_2a3f7d823766e09778f6.mp4 +2025-01-06 22:39:52,815 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 media/videos/eval/4_eval_4_4a5fec9c26b35ac2c239.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 media/videos/eval/1_eval_1_51fb3af7bcef5065307e.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 media/videos/eval/3_eval_3_f961b9a58ac3c44eab41.mp4 +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json media/table/eval/total_success_rate_ci_6_70e918bc0a9fb473bc5b.table.json +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 22:39:52,816 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:39:52,816 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:52,816 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:52,816 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:39:52,817 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:52,817 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:39:52,817 INFO SenderThread:2127250 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:39:52,990 INFO wandb-upload_0:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/requirements.txt +2025-01-06 22:39:52,994 INFO wandb-upload_1:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/wandb-summary.json +2025-01-06 22:39:53,007 INFO wandb-upload_2:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/config.yaml +2025-01-06 22:39:53,044 INFO wandb-upload_4:2127250 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/files/output.log +2025-01-06 22:39:53,117 INFO wandb-upload_3:2127250 [upload_job.py:push():130] Uploaded file /tmp/tmpk6k949v4wandb/er0j79x4-events.out.tfevents.1736220870.leonmkim-ROG-Strix-G15CS-G15CS.2127077.0 +2025-01-06 22:39:53,317 INFO Thread-11 (_thread_body):2127250 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:39:53,317 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,317 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:39:53,317 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,317 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:39:53,318 INFO SenderThread:2127250 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:39:53,319 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,319 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:39:53,319 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:39:53,319 INFO SenderThread:2127250 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:39:53,625 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,688 INFO SenderThread:2127250 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:39:53,688 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:39:53,689 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,689 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,689 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:39:53,689 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,689 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:39:53,689 INFO SenderThread:2127250 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:39:53,689 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:39:53,689 INFO HandlerThread:2127250 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send():379] send: final +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send():379] send: footer +2025-01-06 22:39:53,690 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: defer +2025-01-06 22:39:53,690 INFO SenderThread:2127250 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:39:53,690 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:39:53,691 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:39:53,691 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,691 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:39:53,693 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:39:53,694 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:39:53,755 DEBUG SenderThread:2127250 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:39:53,755 INFO MainThread:2127250 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:39:53,756 INFO MainThread:2127250 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:39:53,756 INFO MainThread:2127250 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:39:53,756 DEBUG HandlerThread:2127250 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:39:53,756 INFO HandlerThread:2127250 [handler.py:finish():882] shutting down handler +2025-01-06 22:39:54,691 INFO WriterThread:2127250 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [sender.py:finish():1615] shutting down sender +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:39:54,755 INFO SenderThread:2127250 [file_pusher.py:join():175] waiting for file pusher diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug.log b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ae1330f46f227b35b6bf386e85f71b63e0ca2ddf --- /dev/null +++ b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:34:29,434 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Configure stats pid to 2127077 +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/settings +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug.log +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/logs/debug-internal.log +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1003_0', 'wandb_notes': '1003_0_req_1364_0restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m'} +2025-01-06 22:34:29,435 INFO MainThread:2127077 [wandb_init.py:init():619] starting backend +2025-01-06 22:34:29,436 INFO MainThread:2127077 [wandb_init.py:init():623] setting up manager +2025-01-06 22:34:29,440 INFO MainThread:2127077 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:34:29,441 INFO MainThread:2127077 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:34:29,453 INFO MainThread:2127077 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:34:29,461 INFO MainThread:2127077 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:34:29,715 INFO MainThread:2127077 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:34:29,757 INFO MainThread:2127077 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:34:29,757 INFO MainThread:2127077 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:34:30,097 INFO MainThread:2127077 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:34:30,098 INFO MainThread:2127077 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:34:30,100 INFO MainThread:2127077 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:34:30,101 INFO MainThread:2127077 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/hxim6a4m/tb, True +2025-01-06 22:34:35,685 INFO MainThread:2127077 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:39:54,757 WARNING MsgRouterThr:2127077 [router.py:message_loop():77] message_loop has been closed diff --git a/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fa7789d4ca1cd1e325a22d218a975de166e3266d Binary files /dev/null and b/hxim6a4m/wandb/run-20250106_223429-hxim6a4m/run-hxim6a4m.wandb differ diff --git a/kxd6yppz/.hydra/config.yaml b/kxd6yppz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16788623643fb2a94ac2150593ce5b0a1ccdd979 --- /dev/null +++ b/kxd6yppz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1003_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kxd6yppz/.hydra/hydra.yaml b/kxd6yppz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4100defb00d1cd3e5f6d91425a91dfda9203da7 --- /dev/null +++ b/kxd6yppz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1003_0/kxd6yppz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kxd6yppz/.hydra/overrides.yaml b/kxd6yppz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kxd6yppz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kxd6yppz/eval_policy.log b/kxd6yppz/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..4840fc47b1d281f4680e61dd616673dc20df2eb0 --- /dev/null +++ b/kxd6yppz/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 22:28:15,031][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:28:15,034][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 22:28:19,234][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/yki2vgk2/episode_rosbags/cam_tf_world.npy b/yki2vgk2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/yki2vgk2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/yki2vgk2/episode_rosbags/depth_K.npy b/yki2vgk2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/yki2vgk2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/yki2vgk2/eval_robot.log b/yki2vgk2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8962013f22d3b4fbaeb6af4c4b2b94e281a36002 --- /dev/null +++ b/yki2vgk2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:26:02,847][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:26:02,850][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( +