diff --git a/.gitattributes b/.gitattributes index 9bd7aef967366e425106c902ee1d108a7a9ca02f..59ebb94aaf589a490eea36cd6be91682023234fe 100644 --- a/.gitattributes +++ b/.gitattributes @@ -149,3 +149,29 @@ q6zm6kn3/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text q6zm6kn3/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text q6zm6kn3/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text q6zm6kn3/episode_rosbags/episode_2_2025-01-06-22-54-15.bag filter=lfs diff=lfs merge=lfs -text +x85ya2ao/episode_rosbags/episode_3_2025-01-07-18-31-27.bag filter=lfs diff=lfs merge=lfs -text +dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 filter=lfs diff=lfs merge=lfs -text +dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 filter=lfs diff=lfs merge=lfs -text +dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 filter=lfs diff=lfs merge=lfs -text +dunmkenr/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dunmkenr/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +dunmkenr/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +x85ya2ao/episode_rosbags/episode_2_2025-01-07-18-30-30.bag filter=lfs diff=lfs merge=lfs -text +up4gjipz/episode_rosbags/episode_1_2025-01-06-20-02-04.bag filter=lfs diff=lfs merge=lfs -text +up4gjipz/episode_rosbags/episode_4_2025-01-06-20-04-42.bag filter=lfs diff=lfs merge=lfs -text +x85ya2ao/episode_rosbags/episode_1_2025-01-07-18-29-41.bag filter=lfs diff=lfs merge=lfs -text +x85ya2ao/episode_rosbags/episode_4_2025-01-07-18-32-14.bag filter=lfs diff=lfs merge=lfs -text +qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +qgwo1435/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q6zm6kn3/episode_rosbags/episode_0_2025-01-06-22-52-30.bag filter=lfs diff=lfs merge=lfs -text +q6zm6kn3/episode_rosbags/episode_1_2025-01-06-22-53-17.bag filter=lfs diff=lfs merge=lfs -text +x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag filter=lfs diff=lfs merge=lfs -text +m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag filter=lfs diff=lfs merge=lfs -text diff --git a/8gt3cc6e/episode_rosbags/cam_tf_world.npy b/8gt3cc6e/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/8gt3cc6e/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/8gt3cc6e/episode_rosbags/depth_K.npy b/8gt3cc6e/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/8gt3cc6e/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/8gt3cc6e/eval_robot.log b/8gt3cc6e/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..22ee051bb144ad2696eb83f282cbb9304dad77fc --- /dev/null +++ b/8gt3cc6e/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:05:03,479][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:05:03,482][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/dunmkenr/.hydra/config.yaml b/dunmkenr/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/dunmkenr/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/dunmkenr/.hydra/hydra.yaml b/dunmkenr/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..482f8c46fcd8989211b8c08a83ee43e23dd55b5c --- /dev/null +++ b/dunmkenr/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/dunmkenr/.hydra/overrides.yaml b/dunmkenr/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/dunmkenr/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/dunmkenr/episode_rosbags/aligned_depth_to_color_K.npy b/dunmkenr/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/dunmkenr/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/dunmkenr/episode_rosbags/color_K.npy b/dunmkenr/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/dunmkenr/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/dunmkenr/eval_video/0_eval.mp4 b/dunmkenr/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb034ba505f1496b76ab9817747b2ddc295db8ad --- /dev/null +++ b/dunmkenr/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a65aa0587775ec8eeac9c2d68f0b44f5a08a547f17a5e561be5264a02ccb2c54 +size 1412628 diff --git a/dunmkenr/eval_video/1_eval.mp4 b/dunmkenr/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df288732c4b876a6d5d0ad348e0766ced6aebdfb --- /dev/null +++ b/dunmkenr/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35882766384911ebc98dc28f5dde920979fbe372ed11964efe5b6e8792fd7522 +size 1402664 diff --git a/dunmkenr/eval_video/2_eval.mp4 b/dunmkenr/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1b1f7b43a1c08574de5a2dbef2be8f91526f148 --- /dev/null +++ b/dunmkenr/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a7f0b0ac6df33894a9b565d2c075f6400b78b032eef18988a93b98e4f36b0d +size 1429312 diff --git a/dunmkenr/eval_video/3_eval.mp4 b/dunmkenr/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/dunmkenr/eval_video/3_eval.mp4 differ diff --git a/dunmkenr/tb/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 b/dunmkenr/tb/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 new file mode 100644 index 0000000000000000000000000000000000000000..849d2e22f8467837523eb08fef9fe189f62d7a85 --- /dev/null +++ b/dunmkenr/tb/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31ec114344ea023eaf7caec93c9ca610f5b16c23da1a14c96c34c0ce5648bbaf +size 2254 diff --git a/dunmkenr/wandb/debug-internal.log b/dunmkenr/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4b8326c9135cb3c4d0fc7eec04f9f07fbbe58de4 --- /dev/null +++ b/dunmkenr/wandb/debug-internal.log @@ -0,0 +1,595 @@ +2025-01-07 18:33:37,263 INFO StreamThr :2214363 [internal.py:wandb_internal():85] W&B internal server running at pid: 2214363, started at: 2025-01-07 18:33:37.262923 +2025-01-07 18:33:37,265 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:33:37,276 INFO WriterThread:2214363 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb +2025-01-07 18:33:37,276 DEBUG SenderThread:2214363 [sender.py:send():379] send: header +2025-01-07 18:33:37,284 DEBUG SenderThread:2214363 [sender.py:send():379] send: run +2025-01-07 18:33:37,561 INFO SenderThread:2214363 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files +2025-01-07 18:33:37,561 INFO SenderThread:2214363 [sender.py:_start_run_threads():1188] run started: dunmkenr with start time 1736292817.263596 +2025-01-07 18:33:37,584 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:33:37,585 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:33:37,659 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:33:37,664 DEBUG HandlerThread:2214363 [system_info.py:__init__():26] System info init +2025-01-07 18:33:37,664 DEBUG HandlerThread:2214363 [system_info.py:__init__():41] System info init done +2025-01-07 18:33:37,664 INFO HandlerThread:2214363 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:33:37,664 INFO HandlerThread:2214363 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:33:37,665 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:33:37,666 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:33:37,667 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:33:37,677 DEBUG HandlerThread:2214363 [system_info.py:probe():152] Probing system +2025-01-07 18:33:37,678 DEBUG HandlerThread:2214363 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:probe():200] Probing system done +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:33:37.677471', 'startedAt': '2025-01-07T23:33:37.253536', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.604874999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.973, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 862.3428268432617}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:33:37,683 INFO HandlerThread:2214363 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:33:37,683 INFO HandlerThread:2214363 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:_save_code():44] Saving code +2025-01-07 18:33:37,685 DEBUG HandlerThread:2214363 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:33:37,685 DEBUG HandlerThread:2214363 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:33:37,714 DEBUG HandlerThread:2214363 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:33:37,714 INFO HandlerThread:2214363 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:33:37,717 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:33:37,717 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:33:37,718 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:33:37,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:33:37,944 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:33:37,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:37,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:37,947 INFO HandlerThread:2214363 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/tb" + save: true +} +control { + relay_id: "14bfcb6d780547a98780854a8ff08246" +} +_info { + stream_id: "dunmkenr" +} + +2025-01-07 18:33:37,947 DEBUG HandlerThread:2214363 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:33:38,006 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/wlxfjaoz-wandb-metadata.json +2025-01-07 18:33:38,012 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/h9or6fha-code/FISH/eval_robot.py +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: telemetry +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: tbrecord +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:33:38,091 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 with policy live +2025-01-07 18:33:38,293 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/iy43ml28-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:33:38,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH +2025-01-07 18:33:38,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code +2025-01-07 18:33:38,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:39,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:40,562 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:40,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:41,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:42,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:42,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:43,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:44,007 DEBUG SenderThread:2214363 [sender.py:send():379] send: config +2025-01-07 18:33:44,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:45,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:46,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:46,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:47,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:47,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:48,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:49,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:50,565 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:50,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:51,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:52,566 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:52,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:52,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:52,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:53,020 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:53,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:54,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:55,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:56,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:57,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:58,646 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:58,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:59,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:00,568 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:00,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:01,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:02,568 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:02,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:03,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:03,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:04,569 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:04,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:05,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:06,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:07,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:07,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:07,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:08,570 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:08,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:09,050 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:09,570 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:34:09,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:10,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:11,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:12,571 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:12,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:13,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:14,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:14,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:15,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:16,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:17,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:18,573 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:18,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:19,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:19,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:20,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:21,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:22,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:22,947 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:22,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:23,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:24,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:25,091 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:25,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:26,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:27,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:28,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:29,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:30,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:30,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:31,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:32,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:33,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:34,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:35,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:36,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:36,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:37,668 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:34:37,668 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:34:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:37,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:37,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:38,301 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:34:38,302 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 with policy now +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:34:38,302 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media +2025-01-07 18:34:38,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos +2025-01-07 18:34:38,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:34:38,598 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/7kq63xfr-media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,889 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/93ynso08-media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:39,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:40,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:40,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:41,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:41,995 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:42,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:43,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:44,580 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:44,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:45,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:46,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:46,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:47,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:34:47,784 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:47,846 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/7wsrrq9c-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:34:47,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:48,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:48,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:49,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:50,582 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:50,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:51,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:52,583 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:52,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:52,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:52,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:53,007 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:53,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:54,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:55,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:56,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:57,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:58,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:58,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:59,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:00,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:01,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:02,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:03,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:03,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:04,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:05,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:06,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:07,669 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:35:07,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:07,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:07,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:08,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:09,036 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:09,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:10,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:11,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:12,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:13,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:14,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:14,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:15,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:16,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:17,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:18,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:19,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:19,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:20,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:21,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:22,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:22,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:22,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:23,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:24,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:25,005 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:25,967 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:26,516 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:35:26,516 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 with policy now +2025-01-07 18:35:26,521 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:35:26,524 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:35:26,524 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:35:26,537 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:35:26,784 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/353ugkom-media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:26,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:27,068 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/3u3j2wox-media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:27,964 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:28,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:28,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:29,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:30,268 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:30,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:31,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:32,596 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:32,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:33,967 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:34,597 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:34,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:35,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:36,114 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:36,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:37,669 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:35:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:37,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:37,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:38,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:39,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:40,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:41,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:42,109 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:42,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:43,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:44,369 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:35:44,369 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:35:44,370 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:35:44,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:35:44,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:35:44,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:45,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:46,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:46,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:47,720 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:47,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:48,602 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:48,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:49,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:50,603 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:50,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:51,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:52,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:52,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:52,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:53,034 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:53,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:54,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:55,971 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:56,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:57,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:58,606 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:58,961 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:58,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:59,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:00,973 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:01,973 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:02,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:03,962 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:03,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:04,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:05,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:06,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:07,670 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:36:07,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:07,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:07,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:08,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:09,094 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:09,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:10,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:11,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:12,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:13,611 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:13,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:14,748 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:14,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:15,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:16,612 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:16,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:17,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:18,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:19,964 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:19,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:20,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:21,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:22,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:22,951 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:22,981 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:23,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:24,049 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:36:24,050 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 with policy now +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:36:24,051 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:36:24,310 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/lz7xqh9p-media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:36:24,920 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/hmnlfwdd-media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,987 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:25,059 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:25,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:25,988 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:26,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:26,988 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:27,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:28,616 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:28,990 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:29,998 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:30,560 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:30,617 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:30,991 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:31,996 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:32,617 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:33,000 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:33,994 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:34,998 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:35,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:36,001 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:37,004 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:37,670 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:36:37,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:37,949 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:38,004 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:39,008 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:40,009 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:41,013 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:41,101 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:42,013 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:43,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:44,017 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:45,014 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:46,016 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:46,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:47,016 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:48,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:49,019 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:50,018 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:51,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:52,025 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:52,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:52,952 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:53,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:53,060 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:54,026 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:55,018 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:56,024 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:57,026 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:58,024 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:58,625 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:58,982 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:59,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:00,025 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:01,030 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:02,029 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:03,034 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:04,045 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:04,981 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:05,050 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:06,047 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:07,051 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:07,671 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:37:07,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:37:07,952 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:37:08,051 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:09,049 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:10,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:10,052 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:11,052 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:11,365 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:37:11,365 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-07 18:37:11,365 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:37:11,366 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:37:11,366 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:37:11,367 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:11,367 DEBUG SenderThread:2214363 [sender.py:send():379] send: telemetry +2025-01-07 18:37:11,393 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:37:11,393 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:37:11,464 DEBUG SenderThread:2214363 [sender.py:send():379] send: exit +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():588] handling runtime: 213 +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():594] send defer +2025-01-07 18:37:11,464 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,464 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:37:11,465 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:37:11,465 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:37:11,465 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:37:11,465 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:37:11,465 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:37:11,465 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:37:11,466 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:37:11,519 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:37:11,520 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:37:11,520 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:37:11,520 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,520 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:37:11,520 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:37:11,520 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:37:11,520 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,521 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:37:11,521 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,521 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:37:11,521 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:37:11,521 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,521 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:37:11,556 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/vccfnft7-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:11,629 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:11,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:11,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:37:11,887 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/zaadvdo4-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:12,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:17,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:17,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:17,378 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,379 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:37:17,379 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:37:17,379 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:17,379 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,379 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:37:17,379 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:37:17,380 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:37:17,382 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:17,382 DEBUG SenderThread:2214363 [sender.py:send():379] send: summary +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:17,386 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:37:17,386 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,387 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:37:17,387 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,387 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:37:17,390 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:17,490 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:37:17,490 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:17,491 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:37:17,491 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:17,491 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,491 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:37:17,491 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,491 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:37:17,631 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:37:17,632 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:18,493 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:18,893 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:37:18,893 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:18,893 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:18,894 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:37:18,894 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:18,894 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:37:18,894 INFO SenderThread:2214363 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:37:18,896 INFO SenderThread:2214363 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:37:18,912 INFO SenderThread:2214363 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:37:18,921 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:37:18,922 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:18,922 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:37:18,922 DEBUG SenderThread:2214363 [sender.py:send():379] send: artifact +2025-01-07 18:37:19,508 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:19,607 INFO wandb-upload_0:2214363 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu22ruqvc +2025-01-07 18:37:19,632 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:19,793 INFO wandb-upload_1:2214363 [upload_job.py:push():88] Uploaded file /tmp/tmpqcc1ire0/wandb-job.json +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:37:20,446 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml config.yaml +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json wandb-summary.json +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt requirements.txt +2025-01-07 18:37:20,635 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:37:20,635 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:37:20,637 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log output.log +2025-01-07 18:37:20,638 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:37:20,642 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:20,642 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:20,642 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:37:20,642 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:37:20,824 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:20,840 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:37:20,877 INFO wandb-upload_4:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:20,908 INFO wandb-upload_2:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt +2025-01-07 18:37:20,937 INFO wandb-upload_3:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/6wfnx6yt-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:37:21,137 INFO Thread-11 (_thread_body):2214363 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:37:21,137 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,137 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:37:21,138 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:37:21,139 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,139 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:37:21,139 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,139 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:37:21,139 INFO SenderThread:2214363 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:37:21,348 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,348 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:37:21,348 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:37:21,348 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,349 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send():379] send: final +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send():379] send: footer +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,349 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:37:21,369 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:21,369 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:21,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:21,386 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:37:21,390 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:37:21,394 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:37:21,394 INFO HandlerThread:2214363 [handler.py:finish():882] shutting down handler +2025-01-07 18:37:22,369 INFO WriterThread:2214363 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb +2025-01-07 18:37:22,369 INFO SenderThread:2214363 [sender.py:finish():1615] shutting down sender +2025-01-07 18:37:22,370 INFO SenderThread:2214363 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:37:22,370 INFO SenderThread:2214363 [file_pusher.py:join():175] waiting for file pusher diff --git a/dunmkenr/wandb/debug.log b/dunmkenr/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bb6b217f035816804916c5b4f7e511cff97690cc --- /dev/null +++ b/dunmkenr/wandb/debug.log @@ -0,0 +1,37 @@ +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Configure stats pid to 2214189 +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/settings +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug.log +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug-internal.log +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr'} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():619] starting backend +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():623] setting up manager +2025-01-07 18:33:37,262 INFO MainThread:2214189 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:33:37,263 INFO MainThread:2214189 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:33:37,275 INFO MainThread:2214189 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:33:37,282 INFO MainThread:2214189 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:33:37,584 INFO MainThread:2214189 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:33:37,652 INFO MainThread:2214189 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:33:37,652 INFO MainThread:2214189 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:33:37,946 INFO MainThread:2214189 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:33:37,947 INFO MainThread:2214189 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/tb, True +2025-01-07 18:33:44,006 INFO MainThread:2214189 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:37:11,365 INFO MainThread:2214189 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/dunmkenr +2025-01-07 18:37:11,367 INFO MainThread:2214189 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-07 18:37:11,384 INFO MainThread:2214189 [wandb_run.py:_restore():2352] restore +2025-01-07 18:37:11,384 INFO MainThread:2214189 [wandb_run.py:_restore():2358] restore done +2025-01-07 18:37:22,400 INFO MainThread:2214189 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:37:22,401 INFO MainThread:2214189 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:37:22,507 INFO MainThread:2214189 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d87cc4d2bfe372909b4d7abbfeed903f05067348 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736292817 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb034ba505f1496b76ab9817747b2ddc295db8ad --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a65aa0587775ec8eeac9c2d68f0b44f5a08a547f17a5e561be5264a02ccb2c54 +size 1412628 diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df288732c4b876a6d5d0ad348e0766ced6aebdfb --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35882766384911ebc98dc28f5dde920979fbe372ed11964efe5b6e8792fd7522 +size 1402664 diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1b1f7b43a1c08574de5a2dbef2be8f91526f148 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0a7f0b0ac6df33894a9b565d2c075f6400b78b032eef18988a93b98e4f36b0d +size 1429312 diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 differ diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..83bb2f5ac7122f7954ed80407df2d328435ce328 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log @@ -0,0 +1,165 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736292824.007550]: resetting environment +[INFO] [1736292824.015437]: cleared current plan +[INFO] [1736292824.016320]: moving to home +[INFO] [1736292825.019029]: reached home +[INFO] [1736292825.020478]: reset action history +[INFO] [1736292827.730419]: environment reset +Starting episode 0 +[INFO] [1736292827.734544]: resetting environment +[INFO] [1736292827.739046]: cleared current plan +[INFO] [1736292827.740942]: moving to home +[INFO] [1736292828.744128]: reached home +[INFO] [1736292828.745747]: reset action history +[INFO] [1736292831.456129]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736292838.646831]: resetting environment +[INFO] [1736292838.651761]: cleared current plan +[INFO] [1736292838.653006]: moving to home +[INFO] [1736292839.655518]: reached home +[INFO] [1736292839.656982]: reset action history +[INFO] [1736292842.416090]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/episode_rosbags/episode_0_2025-01-07-18-34-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2214189.log +started roslaunch server http://158.130.50.37:44971/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2214454] +process[print_text-2]: started with pid [2214478] +process[rosbag_record-3]: started with pid [2214479] +started bagging! +For topic gripper_width: timestamp difference is 100764670 for nearest: 1736292845778796353 - target: 1736292845678031683 at idx 48 +For topic gripper_width: timestamp difference is 104456978 for nearest: 1736292850445464688 - target: 1736292850341007710 at idx 104 +For topic gripper_width: timestamp difference is 108466641 for nearest: 1736292855645487801 - target: 1736292855537021160 at idx 170 +[INFO] [1736292877.259555]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1736292878.062131]: Stored episode 1. +[INFO] [1736292878.062427]: Saving video... +[INFO] [1736292878.260525]: Video saved! +Starting episode 1 +[INFO] [1736292878.279395]: resetting environment +[INFO] [1736292878.291730]: cleared current plan +[INFO] [1736292878.292027]: moving to home +[INFO] [1736292881.994597]: reached home +[INFO] [1736292881.995456]: reset action history +[INFO] [1736292884.707909]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292886.777840]: resetting environment +[INFO] [1736292886.782120]: cleared current plan +[INFO] [1736292886.782303]: moving to home +[INFO] [1736292887.783799]: reached home +[INFO] [1736292887.784204]: reset action history +[INFO] [1736292890.495567]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/episode_rosbags/episode_1_2025-01-07-18-34-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2214189.log +started roslaunch server http://158.130.50.37:42455/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2214614] +process[print_text-5]: started with pid [2214615] +process[rosbag_record-6]: started with pid [2214639] +started bagging! +[INFO] [1736292925.312505]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736292926.069833]: Stored episode 2. +[INFO] [1736292926.070402]: Saving video... +[INFO] [1736292926.479486]: Video saved! +Starting episode 2 +[INFO] [1736292926.525075]: resetting environment +[INFO] [1736292926.560138]: cleared current plan +[INFO] [1736292926.563359]: moving to home +[INFO] [1736292930.267685]: reached home +[INFO] [1736292930.269456]: reset action history +[INFO] [1736292933.067451]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292943.986846]: resetting environment +[INFO] [1736292944.008444]: cleared current plan +[INFO] [1736292944.009399]: moving to home +[INFO] [1736292945.010923]: reached home +[INFO] [1736292945.011426]: reset action history +[INFO] [1736292947.719956]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/episode_rosbags/episode_2_2025-01-07-18-35-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2214189.log +started roslaunch server http://158.130.50.37:33309/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2214768] +process[print_text-8]: started with pid [2214769] +process[rosbag_record-9]: started with pid [2214770] +started bagging! +For topic gripper_width: timestamp difference is -142589012 for nearest: 1736292957078743776 - target: 1736292957221332788 at idx 119 +[rosbag_record-9] process has died [pid 2214770, exit code -11, cmd /opt/ros/noetic/lib/rosbag/record -O /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/episode_rosbags/episode_2_2025-01-07-18-35-47.bag -b 0 /panda/franka_state_controller_custom/franka_states /camera/color/image_raw /camera/aligned_depth_to_color/image_raw /eval_policy/policy_obs /panda/plan /eval_policy/contact_predictor_input /camera/contact_obs /eval_policy/mask_predictor_input /camera/mask_obs /panda/hybrid_impedance_wrench_controller/pose_wrench_desired /panda/franka_gripper/joint_states /panda/franka_gripper/grasp/status /panda/franka_gripper/grasp/goal /panda/franka_gripper/grasp/result /panda/franka_gripper/stop/status /panda/franka_gripper/stop/goal /panda/franka_gripper/stop/result /panda/franka_gripper/move/status /panda/franka_gripper/move/goal /panda/franka_gripper/move/result /tf /tf_static __name:=rosbag_record __log:=/home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/rosbag_record-9.log]. +log file: /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/rosbag_record-9*.log +For topic gripper_width: timestamp difference is 104754962 for nearest: 1736292974745461978 - target: 1736292974640707016 at idx 249 +[INFO] [1736292982.497785]: Storing episode... +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1736292983.297663]: Stored episode 3. +[INFO] [1736292983.305209]: Saving video... +[WARN] [1736292983.883898]: Plan exhausted +[WARN] [1736292983.922650]: Plan exhausted +[WARN] [1736292983.966650]: Plan exhausted +[WARN] [1736292984.005105]: Plan exhausted +[WARN] [1736292984.043282]: Plan exhausted +[INFO] [1736292983.977343]: Video saved! +Starting episode 3 +[INFO] [1736292984.040413]: resetting environment +[INFO] [1736292984.057975]: cleared current plan +[INFO] [1736292984.058230]: moving to home +[INFO] [1736292987.761760]: reached home +[INFO] [1736292987.766644]: reset action history +[INFO] [1736292990.559817]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1736293031.304937]: Saving video... +[INFO] [1736293031.312931]: Video saved! \ No newline at end of file diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9c6d49b0f862060bf24760ba64d7862c2e207bde --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T23:33:37.677471", + "startedAt": "2025-01-07T23:33:37.253536", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4123.604874999999, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.973, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 862.3428268432617 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7653918b54c64869367a77808ae6b42ff0839203 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a65aa0587775ec8eeac9c2d68f0b44f5a08a547f17a5e561be5264a02ccb2c54", "size": 1412628, "path": "media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4"}, "global_step": 266, "_timestamp": 1736292943.9867446, "_runtime": 126.72314858436584, "_step": 5, "eval/1_eval": {"_type": "video-file", "sha256": "35882766384911ebc98dc28f5dde920979fbe372ed11964efe5b6e8792fd7522", "size": 1402664, "path": "media/videos/eval/1_eval_1_35882766384911ebc98d.mp4"}, "eval/num_success": 2.0, "episode": 2.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.5245950222015381, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.3464587330818176, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.3402244746685028, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.3280957341194153, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "d0a7f0b0ac6df33894a9b565d2c075f6400b78b032eef18988a93b98e4f36b0d", "size": 1429312, "path": "media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 213}} \ No newline at end of file diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug-internal.log b/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4b8326c9135cb3c4d0fc7eec04f9f07fbbe58de4 --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug-internal.log @@ -0,0 +1,595 @@ +2025-01-07 18:33:37,263 INFO StreamThr :2214363 [internal.py:wandb_internal():85] W&B internal server running at pid: 2214363, started at: 2025-01-07 18:33:37.262923 +2025-01-07 18:33:37,265 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:33:37,276 INFO WriterThread:2214363 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb +2025-01-07 18:33:37,276 DEBUG SenderThread:2214363 [sender.py:send():379] send: header +2025-01-07 18:33:37,284 DEBUG SenderThread:2214363 [sender.py:send():379] send: run +2025-01-07 18:33:37,561 INFO SenderThread:2214363 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files +2025-01-07 18:33:37,561 INFO SenderThread:2214363 [sender.py:_start_run_threads():1188] run started: dunmkenr with start time 1736292817.263596 +2025-01-07 18:33:37,584 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:33:37,585 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:33:37,659 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:33:37,664 DEBUG HandlerThread:2214363 [system_info.py:__init__():26] System info init +2025-01-07 18:33:37,664 DEBUG HandlerThread:2214363 [system_info.py:__init__():41] System info init done +2025-01-07 18:33:37,664 INFO HandlerThread:2214363 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:33:37,664 INFO HandlerThread:2214363 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:33:37,664 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:33:37,665 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:33:37,666 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:33:37,667 INFO SystemMonitor:2214363 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:33:37,677 DEBUG HandlerThread:2214363 [system_info.py:probe():152] Probing system +2025-01-07 18:33:37,678 DEBUG HandlerThread:2214363 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:probe():200] Probing system done +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:33:37.677471', 'startedAt': '2025-01-07T23:33:37.253536', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.604874999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.973, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 862.3428268432617}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:33:37,683 INFO HandlerThread:2214363 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:33:37,683 INFO HandlerThread:2214363 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:33:37,683 DEBUG HandlerThread:2214363 [system_info.py:_save_code():44] Saving code +2025-01-07 18:33:37,685 DEBUG HandlerThread:2214363 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:33:37,685 DEBUG HandlerThread:2214363 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:33:37,714 DEBUG HandlerThread:2214363 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:33:37,714 INFO HandlerThread:2214363 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:33:37,717 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:33:37,717 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:33:37,718 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:33:37,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:33:37,944 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:33:37,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:37,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:37,947 INFO HandlerThread:2214363 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/tb" + save: true +} +control { + relay_id: "14bfcb6d780547a98780854a8ff08246" +} +_info { + stream_id: "dunmkenr" +} + +2025-01-07 18:33:37,947 DEBUG HandlerThread:2214363 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:33:38,006 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/wlxfjaoz-wandb-metadata.json +2025-01-07 18:33:38,012 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/h9or6fha-code/FISH/eval_robot.py +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: telemetry +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: tbrecord +2025-01-07 18:33:38,090 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:33:38,091 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 with policy live +2025-01-07 18:33:38,293 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/iy43ml28-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:38,563 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:33:38,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH +2025-01-07 18:33:38,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code +2025-01-07 18:33:38,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:39,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:40,562 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:40,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:41,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:42,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:42,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:43,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:44,007 DEBUG SenderThread:2214363 [sender.py:send():379] send: config +2025-01-07 18:33:44,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:45,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:46,564 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:46,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:47,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:47,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:48,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:49,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:50,565 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:50,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:51,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:52,566 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:33:52,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:33:52,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:33:52,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:53,020 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:53,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:54,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:55,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:56,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:57,947 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:58,646 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:33:58,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:33:59,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:00,568 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:00,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:01,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:02,568 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:02,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:03,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:03,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:04,569 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:04,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:05,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:06,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:07,944 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:07,945 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:07,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:08,570 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:08,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:09,050 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:09,570 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:34:09,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:10,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:11,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:12,571 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:12,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:13,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:14,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:14,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:15,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:16,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:17,950 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:18,573 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:18,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:19,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:19,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:20,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:21,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:22,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:22,947 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:22,985 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:23,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:24,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:25,091 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:25,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:26,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:27,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:28,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:29,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:30,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:30,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:31,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:32,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:33,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:34,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:35,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:36,948 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:36,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:37,668 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:34:37,668 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:34:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:37,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:37,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:38,301 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:34:38,302 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 with policy now +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:34:38,302 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:34:38,302 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,578 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media +2025-01-07 18:34:38,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos +2025-01-07 18:34:38,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:34:38,598 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/7kq63xfr-media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,889 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/93ynso08-media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:34:38,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:39,953 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:40,579 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:40,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:41,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:41,995 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:42,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:43,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:44,580 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:44,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:45,954 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:46,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:46,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:47,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:34:47,784 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:47,846 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/7wsrrq9c-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:34:47,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:48,581 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:48,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:49,955 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:50,582 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:50,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:51,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:52,583 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:34:52,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:34:52,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:34:52,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:53,007 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:53,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:54,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:55,956 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:56,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:57,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:58,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:34:58,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:34:59,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:00,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:01,957 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:02,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:03,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:03,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:04,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:05,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:06,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:07,669 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:35:07,945 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:07,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:07,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:08,958 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:09,036 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:09,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:10,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:11,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:12,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:13,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:14,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:14,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:15,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:16,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:17,959 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:18,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:19,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:19,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:20,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:21,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:22,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:22,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:22,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:23,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:24,960 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:25,005 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:25,967 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:26,516 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:35:26,516 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 with policy now +2025-01-07 18:35:26,521 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:35:26,524 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:35:26,524 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:35:26,537 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:26,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:35:26,784 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/353ugkom-media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:26,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:27,068 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/3u3j2wox-media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:35:27,964 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:28,594 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:28,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:29,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:30,268 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:30,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:31,963 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:32,596 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:32,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:33,967 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:34,597 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:34,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:35,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:36,114 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:36,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:37,669 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:35:37,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:37,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:37,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:38,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:39,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:40,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:41,968 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:42,109 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:42,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:43,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:44,369 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:35:44,369 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:35:44,370 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:35:44,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:35:44,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:35:44,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:45,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:46,601 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:46,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:47,720 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:47,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:48,602 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:48,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:49,969 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:50,603 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:50,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:51,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:52,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:35:52,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:35:52,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:53,034 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:53,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:54,970 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:55,971 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:56,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:57,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:58,606 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:35:58,961 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:35:58,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:35:59,972 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:00,973 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:01,973 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:02,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:03,962 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:03,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:04,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:05,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:06,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:07,670 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:36:07,946 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:07,946 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:07,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:08,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:09,094 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:09,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:10,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:11,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:12,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:13,611 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:13,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:14,748 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:14,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:15,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:16,612 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:16,975 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:17,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:18,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:19,964 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:19,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:20,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:21,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:22,951 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:22,951 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:22,981 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:23,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:24,049 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:36:24,050 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 with policy now +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:36:24,050 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:36:24,051 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:36:24,310 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/lz7xqh9p-media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:36:24,920 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/hmnlfwdd-media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:36:24,987 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:25,059 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:25,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:25,988 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:26,615 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:26,988 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:27,989 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:28,616 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:28,990 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:29,998 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:30,560 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:30,617 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:30,991 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:31,996 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:32,617 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:33,000 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:33,994 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:34,998 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:35,974 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:36,001 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:37,004 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:37,670 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:36:37,949 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:37,949 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:38,004 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:39,008 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:40,009 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:41,013 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:41,101 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:42,013 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:43,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:44,017 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:45,014 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:46,016 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:46,976 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:47,016 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:48,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:49,019 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:50,018 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:51,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:52,025 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:52,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:36:52,952 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:36:53,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:53,060 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:54,026 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:55,018 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:56,024 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:57,026 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:58,024 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:36:58,625 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:36:58,982 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:36:59,022 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:00,025 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:01,030 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:02,029 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:03,034 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:04,045 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:04,981 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:05,050 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:06,047 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:07,051 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:07,671 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:37:07,952 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:37:07,952 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:37:08,051 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:09,049 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:10,015 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:10,052 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:11,052 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:11,365 DEBUG SenderThread:2214363 [sender.py:send():379] send: files +2025-01-07 18:37:11,365 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-07 18:37:11,365 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:37:11,366 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:37:11,366 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:37:11,367 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:11,367 DEBUG SenderThread:2214363 [sender.py:send():379] send: telemetry +2025-01-07 18:37:11,393 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:37:11,393 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:37:11,464 DEBUG SenderThread:2214363 [sender.py:send():379] send: exit +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():588] handling runtime: 213 +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:11,464 INFO SenderThread:2214363 [sender.py:send_exit():594] send defer +2025-01-07 18:37:11,464 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,464 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:37:11,465 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:37:11,465 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:37:11,465 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:37:11,465 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:37:11,465 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:37:11,465 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:37:11,465 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:37:11,465 DEBUG SystemMonitor:2214363 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:37:11,466 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:37:11,519 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:37:11,520 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:37:11,520 INFO HandlerThread:2214363 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:37:11,520 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,520 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:37:11,520 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:37:11,520 DEBUG SenderThread:2214363 [sender.py:send():379] send: stats +2025-01-07 18:37:11,520 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,521 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:37:11,521 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:11,521 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:37:11,521 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:37:11,521 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:11,521 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:37:11,556 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/vccfnft7-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:11,629 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:11,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:11,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval +2025-01-07 18:37:11,887 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/zaadvdo4-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:12,630 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:17,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:17,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:17,378 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,379 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:37:17,379 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:37:17,379 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:17,379 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,379 DEBUG SenderThread:2214363 [sender.py:send():379] send: history +2025-01-07 18:37:17,379 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:37:17,380 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:37:17,382 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:17,382 DEBUG SenderThread:2214363 [sender.py:send():379] send: summary +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:37:17,386 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:37:17,386 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:37:17,386 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,387 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:37:17,387 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,387 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:37:17,390 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:37:17,490 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:37:17,490 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:17,491 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:37:17,491 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:17,491 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:17,491 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:37:17,491 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:17,491 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:37:17,631 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:37:17,632 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:18,493 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:18,893 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:37:18,893 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:18,893 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:18,894 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:37:18,894 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:18,894 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:37:18,894 INFO SenderThread:2214363 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:37:18,896 INFO SenderThread:2214363 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:37:18,912 INFO SenderThread:2214363 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:37:18,921 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:37:18,922 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:18,922 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:37:18,922 DEBUG SenderThread:2214363 [sender.py:send():379] send: artifact +2025-01-07 18:37:19,508 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:19,607 INFO wandb-upload_0:2214363 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu22ruqvc +2025-01-07 18:37:19,632 INFO Thread-12 :2214363 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:19,793 INFO wandb-upload_1:2214363 [upload_job.py:push():88] Uploaded file /tmp/tmpqcc1ire0/wandb-job.json +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:37:20,446 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:37:20,446 INFO SenderThread:2214363 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml config.yaml +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json wandb-summary.json +2025-01-07 18:37:20,633 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt requirements.txt +2025-01-07 18:37:20,635 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:37:20,635 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:37:20,637 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log output.log +2025-01-07 18:37:20,638 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 media/videos/eval/1_eval_1_35882766384911ebc98d.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 media/videos/eval/0_eval_0_a65aa0587775ec8eeac9.mp4 +2025-01-07 18:37:20,641 INFO SenderThread:2214363 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 media/videos/eval/2_eval_2_d0a7f0b0ac6df33894a9.mp4 +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:37:20,642 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:20,642 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:20,642 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:37:20,642 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:37:20,642 INFO SenderThread:2214363 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:37:20,824 INFO wandb-upload_1:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/wandb-summary.json +2025-01-07 18:37:20,840 INFO wandb-upload_0:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/config.yaml +2025-01-07 18:37:20,877 INFO wandb-upload_4:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/output.log +2025-01-07 18:37:20,908 INFO wandb-upload_2:2214363 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/files/requirements.txt +2025-01-07 18:37:20,937 INFO wandb-upload_3:2214363 [upload_job.py:push():130] Uploaded file /tmp/tmp9o75ea3twandb/6wfnx6yt-events.out.tfevents.1736292817.leonmkim-ROG-Strix-G15CS-G15CS.2214189.0 +2025-01-07 18:37:21,137 INFO Thread-11 (_thread_body):2214363 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:37:21,137 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,137 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:37:21,138 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:37:21,138 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:37:21,139 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,139 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:37:21,139 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,139 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:37:21,139 INFO SenderThread:2214363 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:37:21,348 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,348 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:37:21,348 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:37:21,348 INFO SenderThread:2214363 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:37:21,348 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:37:21,349 INFO HandlerThread:2214363 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send():379] send: final +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send():379] send: footer +2025-01-07 18:37:21,349 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: defer +2025-01-07 18:37:21,349 INFO SenderThread:2214363 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:37:21,369 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:37:21,369 DEBUG SenderThread:2214363 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:37:21,378 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:37:21,386 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:37:21,390 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:37:21,394 DEBUG HandlerThread:2214363 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:37:21,394 INFO HandlerThread:2214363 [handler.py:finish():882] shutting down handler +2025-01-07 18:37:22,369 INFO WriterThread:2214363 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb +2025-01-07 18:37:22,369 INFO SenderThread:2214363 [sender.py:finish():1615] shutting down sender +2025-01-07 18:37:22,370 INFO SenderThread:2214363 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:37:22,370 INFO SenderThread:2214363 [file_pusher.py:join():175] waiting for file pusher diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug.log b/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bb6b217f035816804916c5b4f7e511cff97690cc --- /dev/null +++ b/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug.log @@ -0,0 +1,37 @@ +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Configure stats pid to 2214189 +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/settings +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:33:37,255 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug.log +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/wandb/run-20250107_183337-dunmkenr/logs/debug-internal.log +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr'} +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():619] starting backend +2025-01-07 18:33:37,256 INFO MainThread:2214189 [wandb_init.py:init():623] setting up manager +2025-01-07 18:33:37,262 INFO MainThread:2214189 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:33:37,263 INFO MainThread:2214189 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:33:37,275 INFO MainThread:2214189 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:33:37,282 INFO MainThread:2214189 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:33:37,584 INFO MainThread:2214189 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:33:37,652 INFO MainThread:2214189 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:33:37,652 INFO MainThread:2214189 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:33:37,945 INFO MainThread:2214189 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:33:37,946 INFO MainThread:2214189 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:33:37,947 INFO MainThread:2214189 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/dunmkenr/tb, True +2025-01-07 18:33:44,006 INFO MainThread:2214189 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:37:11,365 INFO MainThread:2214189 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/dunmkenr +2025-01-07 18:37:11,367 INFO MainThread:2214189 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-07 18:37:11,384 INFO MainThread:2214189 [wandb_run.py:_restore():2352] restore +2025-01-07 18:37:11,384 INFO MainThread:2214189 [wandb_run.py:_restore():2358] restore done +2025-01-07 18:37:22,400 INFO MainThread:2214189 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:37:22,401 INFO MainThread:2214189 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:37:22,507 INFO MainThread:2214189 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb b/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb new file mode 100644 index 0000000000000000000000000000000000000000..16a978b49ffe42e4afab089964aab72eee664cf4 Binary files /dev/null and b/dunmkenr/wandb/run-20250107_183337-dunmkenr/run-dunmkenr.wandb differ diff --git a/m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag b/m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..6695f98c823e9b5a2e6518526f1544e5f96042f9 --- /dev/null +++ b/m386gyxa/episode_rosbags/episode_2_2025-01-06-20-07-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1967d326985b3b484eecd650e6c78c29a35970b89934a2f9b157c891a480daf3 +size 2606381444 diff --git a/q6zm6kn3/episode_rosbags/episode_0_2025-01-06-22-52-30.bag b/q6zm6kn3/episode_rosbags/episode_0_2025-01-06-22-52-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..ecf960fda3321fa5fb208be5176f3e1aa4262398 --- /dev/null +++ b/q6zm6kn3/episode_rosbags/episode_0_2025-01-06-22-52-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7c7f20ad6103baf00058dbf7108c9969ce57eced1f77900c7ddaffcaf043ea2 +size 2616383235 diff --git a/q6zm6kn3/episode_rosbags/episode_1_2025-01-06-22-53-17.bag b/q6zm6kn3/episode_rosbags/episode_1_2025-01-06-22-53-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..79dd27786969c8cef31ebff8c5545eb6066ed504 --- /dev/null +++ b/q6zm6kn3/episode_rosbags/episode_1_2025-01-06-22-53-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19246accc4f19bb49876d5ad2e07a4ccb96171b202ad0aa88e4a6473c765bf9a +size 2593734013 diff --git a/qgwo1435/.hydra/config.yaml b/qgwo1435/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd5530cb9a8f25d6d55135607d04c2061b4351d1 --- /dev/null +++ b/qgwo1435/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1017_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/qgwo1435/.hydra/hydra.yaml b/qgwo1435/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73113117d709e241e697d78143c3c4b3bd9d4eaf --- /dev/null +++ b/qgwo1435/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/qgwo1435/.hydra/overrides.yaml b/qgwo1435/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/qgwo1435/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/qgwo1435/episode_rosbags/aligned_depth_to_color_K.npy b/qgwo1435/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qgwo1435/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qgwo1435/episode_rosbags/cam_tf_world.npy b/qgwo1435/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/qgwo1435/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/qgwo1435/episode_rosbags/color_K.npy b/qgwo1435/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/qgwo1435/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/qgwo1435/episode_rosbags/depth_K.npy b/qgwo1435/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/qgwo1435/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/qgwo1435/eval_robot.log b/qgwo1435/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..24250313dc43bcf498d3d3a474b33ca6f3bc755c --- /dev/null +++ b/qgwo1435/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 16:28:08,977][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:28:08,980][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/qgwo1435/eval_video/0_eval.mp4 b/qgwo1435/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e81d11c5a553cf251d3453e524c5185a20aadb9b --- /dev/null +++ b/qgwo1435/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cb3ba1fd3a94289841ef064645b23acb9c593039ec71da217206239f9d55e5c +size 1550840 diff --git a/qgwo1435/eval_video/1_eval.mp4 b/qgwo1435/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b63c957bc61e162df79cb5eaf70880439df262ae --- /dev/null +++ b/qgwo1435/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9718732bc54ea0f36a225517018e0868e840ac9920f1a8723b6d983408f8f008 +size 1510217 diff --git a/qgwo1435/eval_video/2_eval.mp4 b/qgwo1435/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22c960d8e4cc91f3be32e5fe75a137f1f3c0a379 --- /dev/null +++ b/qgwo1435/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75fd82618aef0bd510b911ea0d8a0cc78ed05b75c55bcea2fc438d4bc80e48a4 +size 1562586 diff --git a/qgwo1435/eval_video/3_eval.mp4 b/qgwo1435/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..594b6d48a08b4d15dac47564d58c8ea0a6ae49c3 --- /dev/null +++ b/qgwo1435/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2751d2990cf3529f7f75cdff1c30a318e18e802b62476bef8eb24401a29d48b4 +size 1528715 diff --git a/qgwo1435/eval_video/4_eval.mp4 b/qgwo1435/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14342571730527ea0b2cba4380801ee21ad7373d --- /dev/null +++ b/qgwo1435/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab8def83cb73c444594a7c19803675dbe5616d3f0ca595d20b510d560aa2cc1 +size 1582048 diff --git a/qgwo1435/tb/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 b/qgwo1435/tb/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 new file mode 100644 index 0000000000000000000000000000000000000000..6974d58212b0d18de468924fe443bbb341f98125 --- /dev/null +++ b/qgwo1435/tb/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a86c9c40bfe47dc2b963446b1e9b188a257462f6b4dc7098fdf737e701383204 +size 6539 diff --git a/qgwo1435/wandb/debug-internal.log b/qgwo1435/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d0255bfc36cc0b9a3b2a16f06e76b7d623c33b9a --- /dev/null +++ b/qgwo1435/wandb/debug-internal.log @@ -0,0 +1,835 @@ +2025-01-07 16:28:14,939 INFO StreamThr :2184937 [internal.py:wandb_internal():85] W&B internal server running at pid: 2184937, started at: 2025-01-07 16:28:14.939523 +2025-01-07 16:28:14,940 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:28:14,953 INFO WriterThread:2184937 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb +2025-01-07 16:28:14,953 DEBUG SenderThread:2184937 [sender.py:send():379] send: header +2025-01-07 16:28:14,961 DEBUG SenderThread:2184937 [sender.py:send():379] send: run +2025-01-07 16:28:15,188 INFO SenderThread:2184937 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files +2025-01-07 16:28:15,188 INFO SenderThread:2184937 [sender.py:_start_run_threads():1188] run started: qgwo1435 with start time 1736285294.939988 +2025-01-07 16:28:15,207 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:28:15,207 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:28:15,283 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:28:15,287 DEBUG HandlerThread:2184937 [system_info.py:__init__():26] System info init +2025-01-07 16:28:15,287 DEBUG HandlerThread:2184937 [system_info.py:__init__():41] System info init done +2025-01-07 16:28:15,287 INFO HandlerThread:2184937 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:28:15,287 INFO SystemMonitor:2184937 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:28:15,288 INFO HandlerThread:2184937 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:28:15,288 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:28:15,289 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:28:15,289 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:28:15,290 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:28:15,291 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:28:15,301 DEBUG HandlerThread:2184937 [system_info.py:probe():152] Probing system +2025-01-07 16:28:15,303 DEBUG HandlerThread:2184937 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:probe():200] Probing system done +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:28:15.302047', 'startedAt': '2025-01-07T21:28:14.925677', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.963, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4489.513, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.155, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 662.953254699707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:28:15,308 INFO HandlerThread:2184937 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:28:15,308 INFO HandlerThread:2184937 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:_save_code():44] Saving code +2025-01-07 16:28:15,313 DEBUG HandlerThread:2184937 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:28:15,313 DEBUG HandlerThread:2184937 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:28:15,373 DEBUG HandlerThread:2184937 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:28:15,374 INFO HandlerThread:2184937 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:28:15,377 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:28:15,377 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:28:15,377 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:28:15,604 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/k1ystv13-wandb-metadata.json +2025-01-07 16:28:15,619 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:28:15,619 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:28:15,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:15,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:15,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:15,623 INFO HandlerThread:2184937 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/tb" + save: true +} +control { + relay_id: "6a745dcc215245ddb00f02e156b63346" +} +_info { + stream_id: "qgwo1435" +} + +2025-01-07 16:28:15,623 DEBUG HandlerThread:2184937 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:28:15,668 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/ryvgde2g-code/FISH/eval_robot.py +2025-01-07 16:28:15,725 DEBUG SenderThread:2184937 [sender.py:send():379] send: telemetry +2025-01-07 16:28:15,725 DEBUG SenderThread:2184937 [sender.py:send():379] send: tbrecord +2025-01-07 16:28:15,726 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:28:15,726 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 with policy live +2025-01-07 16:28:15,909 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/4p42od4d-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code +2025-01-07 16:28:16,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:17,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:18,189 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:18,362 DEBUG SenderThread:2184937 [sender.py:send():379] send: config +2025-01-07 16:28:18,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:19,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:20,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:20,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:20,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:21,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:22,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:22,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:23,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:24,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:24,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:25,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:25,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:26,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:27,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:28,192 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:28,669 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:29,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:30,193 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:30,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:30,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:30,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:30,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:31,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:32,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:33,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:34,194 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:34,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:35,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:36,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:36,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:37,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:38,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:39,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:40,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:41,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:41,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:42,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:43,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:44,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:45,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:45,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:46,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:46,743 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:47,197 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:28:47,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:48,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:49,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:50,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:51,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:52,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:52,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:53,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:54,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:55,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:56,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:57,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:57,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:58,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:59,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:00,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:00,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:00,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:01,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:02,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:02,696 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:03,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:04,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:05,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:06,202 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:06,630 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:07,247 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:29:07,247 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 with policy now +2025-01-07 16:29:07,248 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:29:07,248 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:29:07,248 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:29:07,249 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:29:07,511 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/278etliq-media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:07,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media +2025-01-07 16:29:08,204 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:29:08,259 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:08,520 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/rhf5m25t-media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:08,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:09,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:10,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:11,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:12,204 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:12,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:13,375 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:13,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:14,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:15,291 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:29:15,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:29:15,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:15,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:15,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:16,206 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:16,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:17,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:18,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:18,722 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:19,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:20,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:21,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:22,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:23,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:24,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:24,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:25,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:26,634 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:27,634 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:28,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:29,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:29,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:30,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:30,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:30,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:31,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:32,211 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:29:32,457 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/uw5jodse-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:29:32,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:33,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:34,212 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:34,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:34,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:35,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:36,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:37,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:38,213 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:38,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:39,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:40,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:40,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:41,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:42,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:43,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:44,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:45,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:29:45,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:45,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:45,781 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:46,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:47,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:48,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:49,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:50,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:51,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:51,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:52,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:53,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:54,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:55,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:56,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:56,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:57,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:58,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:59,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:00,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:00,621 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:00,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:01,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:01,798 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:02,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:03,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:04,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:05,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:06,641 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:07,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:07,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:08,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:09,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:10,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:11,194 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:30:11,194 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 with policy now +2025-01-07 16:30:11,195 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:30:11,195 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:30:11,196 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:30:11,207 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:30:11,459 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/sknyoflk-media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:11,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:11,729 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/hmac0ex5-media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:12,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:12,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:13,210 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:13,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:14,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:15,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:30:15,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:15,627 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:15,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:16,225 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:16,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:17,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:18,226 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:18,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:18,993 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:19,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:20,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:21,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:22,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:23,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:24,626 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:24,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:25,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:26,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:27,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:28,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:29,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:29,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:30,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:30,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:30,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:31,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:32,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:33,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:34,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:34,783 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:35,232 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:30:35,473 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/0zir1kyk-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:30:35,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:36,075 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:30:36,075 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:30:36,076 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:30:36,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:36,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:30:36,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:37,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:38,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:38,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:39,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:40,234 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:40,241 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:40,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:41,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:42,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:43,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:44,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:45,293 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:30:45,293 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:45,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:45,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:46,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:47,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:48,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:49,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:50,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:50,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:51,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:52,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:53,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:54,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:55,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:56,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:56,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:57,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:58,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:59,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:00,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:00,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:00,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:01,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:01,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:02,241 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:02,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:03,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:04,242 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:04,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:05,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:06,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:07,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:07,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:08,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:09,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:10,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:11,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:12,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:13,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:13,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:14,660 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:15,293 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:31:15,423 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:31:15,423 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 with policy now +2025-01-07 16:31:15,426 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:31:15,426 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:31:15,427 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:31:15,435 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:31:15,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:15,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:15,656 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:15,674 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/v9l72s9p-media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:31:16,540 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/jmx8c4gh-media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,663 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:17,659 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:18,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:18,658 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:18,746 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:19,663 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:20,247 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:20,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:21,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:22,662 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:23,667 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:23,797 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:24,249 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:24,669 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:25,670 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:26,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:27,666 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:28,666 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:29,431 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:29,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:29,911 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:31:29,911 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:31:29,912 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:31:30,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:30,631 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:30,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:31,672 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:32,252 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:32,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:33,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:34,252 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:34,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:34,784 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:35,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:36,253 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:36,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:37,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:38,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:39,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:40,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:40,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:41,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:42,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:43,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:44,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:45,294 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:31:45,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:45,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:45,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:45,764 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:46,257 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:46,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:47,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:48,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:49,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:50,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:51,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:51,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:52,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:53,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:54,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:55,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:56,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:56,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:57,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:58,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:59,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:00,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:00,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:00,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:01,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:01,782 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:02,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:03,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:04,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:05,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:06,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:07,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:07,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:08,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:09,600 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:32:09,600 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 with policy now +2025-01-07 16:32:09,607 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:32:09,608 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:32:09,608 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:32:09,617 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:32:09,681 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:09,863 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/x45c5h2x-media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,264 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:32:10,576 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/jwax6dbv-media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,684 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:11,682 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:12,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:12,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:12,709 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:13,682 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:14,266 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:14,685 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:15,294 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:32:15,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:15,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:15,683 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:16,266 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:16,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:17,683 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:17,687 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:18,267 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:18,691 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:19,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:20,691 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:21,692 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:22,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:23,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:23,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:24,693 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:25,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:26,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:26,980 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:32:26,980 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:32:26,981 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:32:27,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:32:27,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:32:27,504 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/mgs13i9o-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:32:27,699 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:28,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:28,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:29,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:29,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:30,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:30,625 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:30,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:31,709 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:32,273 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:32,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:33,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:34,273 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:34,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:35,130 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:35,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:36,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:37,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:38,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:39,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:40,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:40,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:41,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:42,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:43,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:44,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:45,295 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:32:45,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:45,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:45,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:45,708 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:46,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:47,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:48,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:49,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:50,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:51,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:51,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:52,279 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:52,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:53,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:54,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:55,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:56,281 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:56,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:57,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:57,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:58,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:59,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:00,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:00,624 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:00,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:01,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:02,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:02,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:03,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:04,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:05,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:06,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:07,718 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:07,761 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:07,837 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:07,837 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 with policy now +2025-01-07 16:33:07,840 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:07,840 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:07,840 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:07,846 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:08,102 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/5hzwlf1o-media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:33:08,555 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/w0r00ykd-media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,721 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:09,722 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:10,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:10,728 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:11,726 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:12,286 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:12,730 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:12,845 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:13,728 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:14,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:15,302 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:33:15,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:15,631 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:15,731 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:16,288 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:16,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:17,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:18,732 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:18,749 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:19,742 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:20,753 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:21,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:22,737 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:23,748 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:24,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:24,755 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:25,747 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:26,751 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:27,028 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:33:27,029 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:33:27,206 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:27,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:27,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval +2025-01-07 16:33:27,514 INFO wandb-upload_0:2184937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpeb4qar7q +2025-01-07 16:33:27,541 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/2jnm22h8-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:27,548 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/l2qbgvhn-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:27,607 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:33:27,614 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:27,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:27,871 INFO SenderThread:2184937 [sender.py:send_request_log_artifact():1525] logged artifact run-qgwo1435-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc3NTgyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1NzQzNw==', 'latestArtifact': None}} +2025-01-07 16:33:27,871 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:27,871 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,871 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:27,872 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,875 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send():379] send: telemetry +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,876 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,877 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,877 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,877 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,877 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,878 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,878 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,879 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,879 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,879 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,879 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,880 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,880 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,881 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,881 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:33:28,109 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/a6noic4j-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:28,244 INFO wandb-upload_2:2184937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpga_q84kn +2025-01-07 16:33:28,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval +2025-01-07 16:33:28,596 INFO SenderThread:2184937 [sender.py:send_request_log_artifact():1525] logged artifact run-qgwo1435-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc3NTg1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1NzQzOA==', 'latestArtifact': None}} +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:28,597 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json with policy now +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:28,597 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:28,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:28,771 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/u9bixu54-media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:29,009 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/eeqlqrb1-media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:29,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:29,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:30,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:30,598 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:30,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:30,624 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:30,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:31,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:32,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:33,329 DEBUG SenderThread:2184937 [sender.py:send():379] send: exit +2025-01-07 16:33:33,329 INFO SenderThread:2184937 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_exit():588] handling runtime: 318 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_exit():594] send defer +2025-01-07 16:33:33,330 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,330 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:33:33,330 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:33:33,330 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:33:33,331 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,331 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:33:33,331 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:33:33,331 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:33:33,331 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:33:33,331 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:33:33,332 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:33:33,332 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:33:33,385 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,385 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:33:33,385 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:33:33,385 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:33:33,385 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,385 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:33:33,386 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,386 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:33:33,386 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:33:33,386 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,386 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:33:34,295 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:34,295 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:33:40,060 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,060 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:33:40,061 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:33:40,061 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:40,061 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,061 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:40,061 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:33:40,062 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:40,064 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:40,064 DEBUG SenderThread:2184937 [sender.py:send():379] send: summary +2025-01-07 16:33:40,068 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:40,069 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:33:40,069 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,069 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:33:40,069 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:33:40,076 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:40,201 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:33:40,201 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,201 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:33:40,201 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,201 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:33:40,297 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:33:40,298 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:40,331 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:33:43,390 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:43,390 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:43,390 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:33:43,390 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:33:43,394 INFO SenderThread:2184937 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:33:43,399 INFO SenderThread:2184937 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:33:43,413 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:33:43,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:43,413 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:33:43,413 DEBUG SenderThread:2184937 [sender.py:send():379] send: artifact +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'versionIndex': 431}}} +2025-01-07 16:33:43,633 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml config.yaml +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json wandb-summary.json +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt requirements.txt +2025-01-07 16:33:44,301 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:33:44,301 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:44,303 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log output.log +2025-01-07 16:33:44,306 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:33:44,307 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:33:44,308 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,308 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:33:44,308 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,309 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:33:44,309 INFO SenderThread:2184937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:33:44,332 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,332 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:44,471 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt +2025-01-07 16:33:44,498 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:33:44,515 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:44,517 INFO wandb-upload_4:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:44,583 INFO wandb-upload_3:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/li31m1s1-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:44,783 INFO Thread-11 (_thread_body):2184937 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:33:44,784 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,784 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:33:44,784 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,784 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:33:44,784 INFO SenderThread:2184937 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:33:44,785 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:33:44,785 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,785 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:33:44,786 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,786 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:33:44,786 INFO SenderThread:2184937 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:33:44,973 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,973 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:33:44,973 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:33:44,973 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,974 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send():379] send: final +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send():379] send: footer +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,974 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:33:44,974 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,975 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:33:44,975 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,975 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:44,975 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:33:44,976 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:33:44,977 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:33:44,978 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:45,028 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:45,028 INFO MainThread:2184937 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:33:45,029 INFO MainThread:2184937 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:33:45,029 INFO MainThread:2184937 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:33:45,029 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:33:45,029 INFO HandlerThread:2184937 [handler.py:finish():882] shutting down handler +2025-01-07 16:33:45,975 INFO WriterThread:2184937 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [sender.py:finish():1615] shutting down sender +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [file_pusher.py:join():175] waiting for file pusher diff --git a/qgwo1435/wandb/debug.log b/qgwo1435/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5c0c0e12419251b6b5d5a901f0856f2c1b967f31 --- /dev/null +++ b/qgwo1435/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Configure stats pid to 2184761 +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/settings +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug.log +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug-internal.log +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435'} +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():619] starting backend +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():623] setting up manager +2025-01-07 16:28:14,938 INFO MainThread:2184761 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:28:14,939 INFO MainThread:2184761 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:28:14,952 INFO MainThread:2184761 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:28:14,959 INFO MainThread:2184761 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:28:15,207 INFO MainThread:2184761 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:28:15,277 INFO MainThread:2184761 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:28:15,278 INFO MainThread:2184761 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:28:15,622 INFO MainThread:2184761 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:28:15,623 INFO MainThread:2184761 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/tb, True +2025-01-07 16:28:18,361 INFO MainThread:2184761 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:33:46,030 WARNING MsgRouterThr:2184761 [router.py:message_loop():77] message_loop has been closed diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf84993aeaa04f5de9cb415217093a69ae1a3c1f --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1017_0' +wandb_notes: + desc: null + value: 1017_0_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736285294 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b5c2eb3589951b58f68feeb6f946614bde198d --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[28, 40, 0.7, 0.5128048360322137, 0.8471510707637759, 0.5511939823525085, 0.833525702290587, 0.5756549462524738, 0.8042996823179089, 0.5930245779268659, 0.7932315337816525]]} \ No newline at end of file diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e81d11c5a553cf251d3453e524c5185a20aadb9b --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cb3ba1fd3a94289841ef064645b23acb9c593039ec71da217206239f9d55e5c +size 1550840 diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b63c957bc61e162df79cb5eaf70880439df262ae --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9718732bc54ea0f36a225517018e0868e840ac9920f1a8723b6d983408f8f008 +size 1510217 diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..22c960d8e4cc91f3be32e5fe75a137f1f3c0a379 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75fd82618aef0bd510b911ea0d8a0cc78ed05b75c55bcea2fc438d4bc80e48a4 +size 1562586 diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..594b6d48a08b4d15dac47564d58c8ea0a6ae49c3 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2751d2990cf3529f7f75cdff1c30a318e18e802b62476bef8eb24401a29d48b4 +size 1528715 diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14342571730527ea0b2cba4380801ee21ad7373d --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab8def83cb73c444594a7c19803675dbe5616d3f0ca595d20b510d560aa2cc1 +size 1582048 diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e793646da807b734322c4f02593894ba94ffd481 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736285298.362460]: resetting environment +[INFO] [1736285298.368671]: cleared current plan +[INFO] [1736285298.369694]: moving to home +[INFO] [1736285299.371968]: reached home +[INFO] [1736285299.373146]: reset action history +[INFO] [1736285302.083679]: environment reset +Starting episode 0 +[INFO] [1736285302.089372]: resetting environment +[INFO] [1736285302.095808]: cleared current plan +[INFO] [1736285302.096740]: moving to home +[INFO] [1736285303.099139]: reached home +[INFO] [1736285303.100381]: reset action history +[INFO] [1736285305.810494]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736285307.609717]: resetting environment +[INFO] [1736285307.615813]: cleared current plan +[INFO] [1736285307.617840]: moving to home +[INFO] [1736285308.621382]: reached home +[INFO] [1736285308.623721]: reset action history +[INFO] [1736285311.331659]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/episode_rosbags/episode_0_2025-01-07-16-28-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2184761.log +started roslaunch server http://158.130.50.37:36571/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2185026] +process[print_text-2]: started with pid [2185050] +process[rosbag_record-3]: started with pid [2185051] +started bagging! +For topic gripper_width: timestamp difference is 107369735 for nearest: 1736285343845530345 - target: 1736285343738160610 at idx 249 +[INFO] [1736285346.177212]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736285346.996765]: Stored episode 1. +[INFO] [1736285346.997002]: Saving video... +[INFO] [1736285347.225407]: Video saved! +Starting episode 1 +[INFO] [1736285347.248969]: resetting environment +[INFO] [1736285347.256973]: cleared current plan +[INFO] [1736285347.257583]: moving to home +[INFO] [1736285350.663202]: reached home +[INFO] [1736285350.664202]: reset action history +[INFO] [1736285353.375000]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285371.660868]: resetting environment +[INFO] [1736285371.668316]: cleared current plan +[INFO] [1736285371.668788]: moving to home +[INFO] [1736285372.671056]: reached home +[INFO] [1736285372.671942]: reset action history +[INFO] [1736285375.379179]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/episode_rosbags/episode_1_2025-01-07-16-29-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2184761.log +started roslaunch server http://158.130.50.37:44893/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2185175] +process[print_text-5]: started with pid [2185176] +process[rosbag_record-6]: started with pid [2185177] +started bagging! +[INFO] [1736285410.203491]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736285410.920125]: Stored episode 2. +[INFO] [1736285410.920391]: Saving video... +[INFO] [1736285411.176709]: Video saved! +Starting episode 2 +[INFO] [1736285411.198251]: resetting environment +[INFO] [1736285411.209375]: cleared current plan +[INFO] [1736285411.209596]: moving to home +[INFO] [1736285414.212842]: reached home +[INFO] [1736285414.216261]: reset action history +[INFO] [1736285416.949016]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285435.044930]: resetting environment +[INFO] [1736285435.059344]: cleared current plan +[INFO] [1736285435.059544]: moving to home +[INFO] [1736285436.061045]: reached home +[INFO] [1736285436.061499]: reset action history +[INFO] [1736285438.770449]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/episode_rosbags/episode_2_2025-01-07-16-30-38.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2184761.log +started roslaunch server http://158.130.50.37:41589/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2185362] +process[print_text-8]: started with pid [2185363] +process[rosbag_record-9]: started with pid [2185364] +started bagging! +For topic gripper_width: timestamp difference is 101605142 for nearest: 1736285460345536197 - target: 1736285460243931055 at idx 249 +For topic gripper_width: timestamp difference is 102598271 for nearest: 1736285461645497403 - target: 1736285461542899132 at idx 249 +[INFO] [1736285473.920898]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736285474.730015]: Stored episode 3. +[INFO] [1736285474.731128]: Saving video... +[INFO] [1736285475.375031]: Video saved! +[WARN] [1736285475.252589]: Plan exhausted +[WARN] [1736285475.291808]: Plan exhausted +[WARN] [1736285475.335137]: Plan exhausted +[WARN] [1736285475.370521]: Plan exhausted +[WARN] [1736285475.413448]: Plan exhausted +Starting episode 3 +[INFO] [1736285475.430489]: resetting environment +[INFO] [1736285475.442463]: cleared current plan +[INFO] [1736285475.442908]: moving to home +[INFO] [1736285478.949430]: reached home +[INFO] [1736285478.953439]: reset action history +[INFO] [1736285481.753671]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285489.452631]: resetting environment +[INFO] [1736285489.484675]: cleared current plan +[INFO] [1736285489.488928]: moving to home +[INFO] [1736285490.495721]: reached home +[INFO] [1736285490.504491]: reset action history +[INFO] [1736285493.274966]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/episode_rosbags/episode_3_2025-01-07-16-31-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2184761.log +started roslaunch server http://158.130.50.37:36589/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2185504] +process[print_text-11]: started with pid [2185505] +process[rosbag_record-12]: started with pid [2185506] +started bagging! +For topic gripper_width: timestamp difference is 103995493 for nearest: 1736285504745568684 - target: 1736285504641573191 at idx 146 +[INFO] [1736285528.384448]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736285529.208511]: Stored episode 4. +[INFO] [1736285529.210701]: Saving video... +[INFO] [1736285529.539537]: Video saved! +Starting episode 4 +[INFO] [1736285529.609360]: resetting environment +[INFO] [1736285529.642209]: cleared current plan +[INFO] [1736285529.643275]: moving to home +[INFO] [1736285532.651349]: reached home +[INFO] [1736285532.670291]: reset action history +[INFO] [1736285535.458256]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736285546.322863]: resetting environment +[INFO] [1736285546.357928]: cleared current plan +[INFO] [1736285546.361023]: moving to home +[INFO] [1736285547.365109]: reached home +[INFO] [1736285547.368063]: reset action history +[INFO] [1736285550.143125]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/episode_rosbags/episode_4_2025-01-07-16-32-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2184761.log +started roslaunch server http://158.130.50.37:37009/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2185648] +process[print_text-14]: started with pid [2185649] +process[rosbag_record-15]: started with pid [2185650] +started bagging! +For topic gripper_width: timestamp difference is -119933259 for nearest: 1736285570078895438 - target: 1736285570198828697 at idx 249 +For topic gripper_width: timestamp difference is -112850851 for nearest: 1736285573478867107 - target: 1736285573591717958 at idx 249 +[INFO] [1736285586.671393]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736285587.463697]: Stored episode 5. +[INFO] [1736285587.465919]: Saving video... +[INFO] [1736285587.754412]: Video saved! +[INFO] [1736285587.842314]: resetting environment +[INFO] [1736285587.883902]: cleared current plan +[INFO] [1736285587.892853]: moving to home +[INFO] [1736285590.799375]: reached home +[INFO] [1736285590.809459]: reset action history +[INFO] [1736285593.666779]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e297d78a9e45a47e20366fb729dc62da2a5aea72 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T21:28:15.302047", + "startedAt": "2025-01-07T21:28:14.925677", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.963, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4489.513, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.155, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 662.953254699707 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..17fefadb84964a89d139f0bc112e4685219931d2 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "2cb3ba1fd3a94289841ef064645b23acb9c593039ec71da217206239f9d55e5c", "size": 1550840, "path": "media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4"}, "global_step": 665, "_timestamp": 1736285607.5312939, "_runtime": 312.59130597114563, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9718732bc54ea0f36a225517018e0868e840ac9920f1a8723b6d983408f8f008", "size": 1510217, "path": "media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4"}, "eval/num_success": 1.0, "episode": 40.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "75fd82618aef0bd510b911ea0d8a0cc78ed05b75c55bcea2fc438d4bc80e48a4", "size": 1562586, "path": "media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2751d2990cf3529f7f75cdff1c30a318e18e802b62476bef8eb24401a29d48b4", "size": 1528715, "path": "media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7ab8def83cb73c444594a7c19803675dbe5616d3f0ca595d20b510d560aa2cc1", "size": 1582048, "path": "media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://wz79qhyx0h199efmikb1z2vp4peqq0n83z8t6yrraibeokksl6q0wwicv33hsjawfkum8x5wti2cpq5wdkzcy1qyxb99c0zcha2z8wx3caf73ad1usa2qrn3ehjnpokv/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://61ngjmkgiew16mzjrir5dec46w3y89anu2cx12mb0ubc89nnxatlscc4br3hg8kqhyxkrnj6y5wrg2ai4b19vmkxc3d3xe3b5cifvcnnod4i5aoynm8kk2ws4cu3jxo2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 28, "total_num_episodes": 40, "total_success_rate": 0.7, "total_success_rate_lb_0.01": 0.5128048360322137, "total_success_rate_ub_0.01": 0.8471510707637759, "total_success_rate_lb_0.025": 0.5511939823525085, "total_success_rate_ub_0.025": 0.833525702290587, "total_success_rate_lb_0.05": 0.5756549462524738, "total_success_rate_ub_0.05": 0.8042996823179089, "total_success_rate_lb_0.1": 0.5930245779268659, "total_success_rate_ub_0.1": 0.7932315337816525, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3f23270287b36641e4b1914ed8397e2719343945e56176dfad94d32c9549dfff", "size": 504, "artifact_path": "wandb-client-artifact://bub45kx2dci4xiddlzjca1eioqixbvchakvr61ukxzceub5x8k4s73p9b6p1ggit07rmvo38zdq4gbt78utwcbuh7lqoqhapzerx39ih7vpg9929c4apub0jaarvvtvb/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://phs598q88s4pxgs595m31p1gm99jn37wleletdyuwfadjl319ltupejzkod2z13m21p3idqtopipak51ybgts1j131j8t7jrgtqjatf24icq7xli1epqny3blp3wapin:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 318}, "eval/total_success_rate": 0.699999988079071, "eval/total_success_rate_lb_0.01": 0.5128048062324524, "eval/total_success_rate_ub_0.01": 0.8471510410308838, "eval/total_success_rate_lb_0.025": 0.5511939525604248, "eval/total_success_rate_ub_0.025": 0.8335257172584534, "eval/total_success_rate_lb_0.05": 0.575654923915863, "eval/total_success_rate_ub_0.05": 0.8042996525764465, "eval/total_success_rate_lb_0.1": 0.5930245518684387, "eval/total_success_rate_ub_0.1": 0.7932315468788147} \ No newline at end of file diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug-internal.log b/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d0255bfc36cc0b9a3b2a16f06e76b7d623c33b9a --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug-internal.log @@ -0,0 +1,835 @@ +2025-01-07 16:28:14,939 INFO StreamThr :2184937 [internal.py:wandb_internal():85] W&B internal server running at pid: 2184937, started at: 2025-01-07 16:28:14.939523 +2025-01-07 16:28:14,940 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:28:14,953 INFO WriterThread:2184937 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb +2025-01-07 16:28:14,953 DEBUG SenderThread:2184937 [sender.py:send():379] send: header +2025-01-07 16:28:14,961 DEBUG SenderThread:2184937 [sender.py:send():379] send: run +2025-01-07 16:28:15,188 INFO SenderThread:2184937 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files +2025-01-07 16:28:15,188 INFO SenderThread:2184937 [sender.py:_start_run_threads():1188] run started: qgwo1435 with start time 1736285294.939988 +2025-01-07 16:28:15,207 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:28:15,207 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:28:15,283 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:28:15,287 DEBUG HandlerThread:2184937 [system_info.py:__init__():26] System info init +2025-01-07 16:28:15,287 DEBUG HandlerThread:2184937 [system_info.py:__init__():41] System info init done +2025-01-07 16:28:15,287 INFO HandlerThread:2184937 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:28:15,287 INFO SystemMonitor:2184937 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:28:15,288 INFO HandlerThread:2184937 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:28:15,288 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:28:15,289 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:28:15,289 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:28:15,290 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:28:15,291 INFO SystemMonitor:2184937 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:28:15,301 DEBUG HandlerThread:2184937 [system_info.py:probe():152] Probing system +2025-01-07 16:28:15,303 DEBUG HandlerThread:2184937 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:probe():200] Probing system done +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:28:15.302047', 'startedAt': '2025-01-07T21:28:14.925677', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.963, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4489.513, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.155, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 662.953254699707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:28:15,308 INFO HandlerThread:2184937 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:28:15,308 INFO HandlerThread:2184937 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:28:15,308 DEBUG HandlerThread:2184937 [system_info.py:_save_code():44] Saving code +2025-01-07 16:28:15,313 DEBUG HandlerThread:2184937 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:28:15,313 DEBUG HandlerThread:2184937 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:28:15,373 DEBUG HandlerThread:2184937 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:28:15,374 INFO HandlerThread:2184937 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:28:15,377 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:28:15,377 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:28:15,377 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:28:15,604 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/k1ystv13-wandb-metadata.json +2025-01-07 16:28:15,619 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:28:15,619 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:28:15,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:15,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:15,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:15,623 INFO HandlerThread:2184937 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/tb" + save: true +} +control { + relay_id: "6a745dcc215245ddb00f02e156b63346" +} +_info { + stream_id: "qgwo1435" +} + +2025-01-07 16:28:15,623 DEBUG HandlerThread:2184937 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:28:15,668 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/ryvgde2g-code/FISH/eval_robot.py +2025-01-07 16:28:15,725 DEBUG SenderThread:2184937 [sender.py:send():379] send: telemetry +2025-01-07 16:28:15,725 DEBUG SenderThread:2184937 [sender.py:send():379] send: tbrecord +2025-01-07 16:28:15,726 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:28:15,726 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 with policy live +2025-01-07 16:28:15,909 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/4p42od4d-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt +2025-01-07 16:28:16,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH +2025-01-07 16:28:16,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code +2025-01-07 16:28:16,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:17,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:18,189 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:18,362 DEBUG SenderThread:2184937 [sender.py:send():379] send: config +2025-01-07 16:28:18,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:19,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:20,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:20,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:20,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:21,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:22,190 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:22,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:23,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:24,191 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:24,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:25,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:25,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:26,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:27,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:28,192 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:28,669 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:29,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:30,193 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:30,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:30,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:30,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:30,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:31,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:32,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:33,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:34,194 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:28:34,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:35,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:36,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:36,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:37,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:38,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:39,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:40,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:41,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:41,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:42,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:43,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:44,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:45,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:28:45,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:28:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:46,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:46,743 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:47,197 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:28:47,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:48,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:49,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:50,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:51,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:52,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:52,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:53,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:54,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:55,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:56,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:57,621 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:28:57,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:58,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:28:59,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:00,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:00,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:00,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:01,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:02,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:02,696 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:03,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:04,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:05,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:06,202 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:06,630 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:07,247 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:29:07,247 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 with policy now +2025-01-07 16:29:07,248 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:29:07,248 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:29:07,248 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:29:07,249 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:29:07,511 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/278etliq-media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:07,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos +2025-01-07 16:29:08,203 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media +2025-01-07 16:29:08,204 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:29:08,259 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:08,520 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/rhf5m25t-media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:29:08,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:09,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:10,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:11,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:12,204 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:12,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:13,375 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:13,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:14,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:15,291 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:29:15,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:29:15,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:15,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:15,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:16,206 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:16,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:17,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:18,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:18,722 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:19,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:20,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:21,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:22,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:23,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:24,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:24,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:25,633 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:26,634 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:27,634 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:28,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:29,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:29,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:30,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:30,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:30,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:31,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:32,211 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:29:32,457 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/uw5jodse-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:29:32,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:33,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:34,212 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:34,635 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:34,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:35,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:36,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:37,636 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:38,213 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:29:38,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:39,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:40,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:40,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:41,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:42,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:43,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:44,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:45,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:29:45,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:29:45,620 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:29:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:45,781 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:46,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:47,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:48,637 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:49,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:50,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:51,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:51,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:52,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:53,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:54,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:55,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:56,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:29:56,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:57,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:58,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:29:59,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:00,620 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:00,621 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:00,661 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:01,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:01,798 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:02,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:03,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:04,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:05,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:06,641 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:07,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:07,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:08,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:09,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:10,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:11,194 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:30:11,194 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 with policy now +2025-01-07 16:30:11,195 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:30:11,195 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:30:11,196 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:30:11,207 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:11,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:30:11,459 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/sknyoflk-media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:11,642 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:11,729 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/hmac0ex5-media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:30:12,224 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:12,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:13,210 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:13,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:14,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:15,292 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:30:15,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:15,627 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:15,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:16,225 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:16,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:17,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:18,226 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:18,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:18,993 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:19,644 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:20,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:21,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:22,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:23,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:24,626 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:24,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:25,645 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:26,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:27,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:28,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:29,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:29,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:30,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:30,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:30,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:31,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:32,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:33,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:34,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:34,783 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:35,232 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:30:35,473 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/0zir1kyk-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:30:35,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:36,075 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:30:36,075 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:30:36,076 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:30:36,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:36,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:30:36,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:37,647 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:38,233 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:38,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:39,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:40,234 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:30:40,241 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:40,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:41,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:42,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:43,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:44,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:45,293 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:30:45,293 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:45,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:30:45,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:30:45,665 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:46,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:47,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:48,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:49,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:50,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:50,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:51,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:52,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:53,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:54,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:55,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:56,627 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:30:56,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:57,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:58,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:30:59,650 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:00,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:00,622 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:00,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:01,648 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:01,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:02,241 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:02,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:03,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:04,242 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:04,651 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:05,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:06,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:07,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:07,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:08,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:09,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:10,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:11,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:12,652 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:13,628 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:13,653 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:14,660 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:15,293 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:31:15,423 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:31:15,423 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 with policy now +2025-01-07 16:31:15,426 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:31:15,426 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:31:15,427 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:31:15,435 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:31:15,622 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:15,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:15,656 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:15,674 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/v9l72s9p-media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:31:16,540 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/jmx8c4gh-media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:31:16,663 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:17,659 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:18,246 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:18,658 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:18,746 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:19,663 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:20,247 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:20,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:21,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:22,662 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:23,667 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:23,797 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:24,249 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:24,669 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:25,670 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:26,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:27,666 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:28,666 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:29,431 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:29,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:29,911 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:31:29,911 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:31:29,912 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:31:30,251 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:31:30,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:30,631 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:30,668 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:31,672 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:32,252 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:32,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:33,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:34,252 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:34,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:34,784 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:35,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:36,253 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:36,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:37,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:38,673 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:39,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:40,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:40,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:41,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:42,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:43,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:44,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:45,294 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:31:45,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:31:45,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:31:45,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:45,764 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:46,257 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:31:46,674 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:47,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:48,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:49,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:50,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:51,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:51,675 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:52,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:53,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:54,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:55,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:56,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:31:56,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:57,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:58,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:31:59,676 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:00,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:00,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:00,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:01,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:01,782 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:02,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:03,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:04,678 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:05,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:06,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:07,632 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:07,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:08,679 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:09,600 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:32:09,600 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 with policy now +2025-01-07 16:32:09,607 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:32:09,608 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:32:09,608 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:32:09,617 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:32:09,681 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:09,863 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/x45c5h2x-media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,264 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:32:10,576 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/jwax6dbv-media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:32:10,684 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:11,682 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:12,265 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:12,646 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:12,709 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:13,682 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:14,266 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:14,685 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:15,294 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:32:15,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:15,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:15,683 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:16,266 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:16,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:17,683 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:17,687 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:18,267 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:18,691 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:19,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:20,691 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:21,692 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:22,689 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:23,638 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:23,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:24,693 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:25,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:26,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:26,980 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:32:26,980 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:32:26,981 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:32:27,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:32:27,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:32:27,504 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/mgs13i9o-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:32:27,699 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:28,271 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:28,697 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:29,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:29,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:30,625 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:30,625 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:30,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:31,709 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:32,273 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:32,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:33,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:34,273 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:34,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:35,130 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:35,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:36,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:37,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:38,702 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:39,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:40,639 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:40,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:41,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:42,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:43,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:44,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:45,295 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:32:45,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:32:45,623 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:32:45,703 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:45,708 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:46,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:47,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:48,704 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:49,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:50,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:51,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:51,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:52,279 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:52,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:53,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:54,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:55,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:56,281 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:32:56,705 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:57,640 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:32:57,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:58,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:32:59,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:00,623 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:00,624 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:00,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:01,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:02,695 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:02,706 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:03,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:04,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:05,707 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:06,713 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:07,718 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:07,761 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:07,837 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:07,837 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 with policy now +2025-01-07 16:33:07,840 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:07,840 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:07,840 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:07,846 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:08,102 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/5hzwlf1o-media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval +2025-01-07 16:33:08,555 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/w0r00ykd-media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:08,721 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:09,722 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:10,285 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:10,728 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:11,726 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:12,286 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:12,730 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:12,845 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:13,728 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:14,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:15,302 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:33:15,631 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:15,631 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:15,731 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:16,288 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:16,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:17,733 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:18,732 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:18,749 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:19,742 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:20,753 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:21,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:22,737 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:23,748 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:24,649 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:24,755 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:25,747 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:26,751 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:27,028 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:33:27,029 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:33:27,206 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:27,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:27,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table +2025-01-07 16:33:27,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval +2025-01-07 16:33:27,514 INFO wandb-upload_0:2184937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpeb4qar7q +2025-01-07 16:33:27,541 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/2jnm22h8-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:27,548 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/l2qbgvhn-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:27,607 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:33:27,614 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:33:27,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:27,871 INFO SenderThread:2184937 [sender.py:send_request_log_artifact():1525] logged artifact run-qgwo1435-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc3NTgyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1NzQzNw==', 'latestArtifact': None}} +2025-01-07 16:33:27,871 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:27,871 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,871 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:27,872 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:27,872 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,875 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send():379] send: telemetry +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,876 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,876 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,877 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,877 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,877 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,877 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,878 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,878 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,878 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,879 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,879 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,879 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,879 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,880 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,880 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,880 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:27,881 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:27,881 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:33:28,109 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/a6noic4j-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:28,244 INFO wandb-upload_2:2184937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpga_q84kn +2025-01-07 16:33:28,292 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:28,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval +2025-01-07 16:33:28,596 INFO SenderThread:2184937 [sender.py:send_request_log_artifact():1525] logged artifact run-qgwo1435-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTc3NTg1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY1NzQzOA==', 'latestArtifact': None}} +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send():379] send: files +2025-01-07 16:33:28,597 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json with policy now +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:28,597 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:28,597 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:28,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:28,771 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/u9bixu54-media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:29,009 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/eeqlqrb1-media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:29,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:29,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:30,293 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:30,598 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:30,624 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:33:30,624 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:33:30,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:31,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:32,741 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:33,329 DEBUG SenderThread:2184937 [sender.py:send():379] send: exit +2025-01-07 16:33:33,329 INFO SenderThread:2184937 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_exit():588] handling runtime: 318 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_exit():594] send defer +2025-01-07 16:33:33,330 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,330 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:33:33,330 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:33:33,330 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:33:33,330 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:33:33,331 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,331 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:33:33,331 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:33:33,331 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:33:33,331 INFO HandlerThread:2184937 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:33:33,331 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:33:33,331 DEBUG SystemMonitor:2184937 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:33:33,332 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:33:33,332 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:33:33,384 INFO HandlerThread:2184937 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:33:33,385 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,385 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:33:33,385 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:33:33,385 DEBUG SenderThread:2184937 [sender.py:send():379] send: stats +2025-01-07 16:33:33,385 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,385 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:33:33,386 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:33,386 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:33:33,386 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:33:33,386 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:33,386 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:33:34,295 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:34,295 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:40,059 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:33:40,060 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,060 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:33:40,061 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:33:40,061 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:40,061 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,061 DEBUG SenderThread:2184937 [sender.py:send():379] send: history +2025-01-07 16:33:40,061 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:33:40,062 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:33:40,064 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:40,064 DEBUG SenderThread:2184937 [sender.py:send():379] send: summary +2025-01-07 16:33:40,068 INFO SenderThread:2184937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:33:40,069 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:33:40,069 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,069 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:33:40,069 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,069 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:33:40,076 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:33:40,201 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:33:40,201 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:40,201 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:33:40,201 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:40,201 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:33:40,297 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:33:40,298 INFO Thread-12 :2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:40,331 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:33:43,390 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:43,390 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:43,390 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:33:43,390 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:33:43,390 INFO SenderThread:2184937 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:33:43,394 INFO SenderThread:2184937 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:33:43,399 INFO SenderThread:2184937 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:33:43,413 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:33:43,413 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:43,413 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:33:43,413 DEBUG SenderThread:2184937 [sender.py:send():379] send: artifact +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc1Nzg0Ng==', 'versionIndex': 431}}} +2025-01-07 16:33:43,633 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:33:43,633 INFO SenderThread:2184937 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml config.yaml +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json wandb-summary.json +2025-01-07 16:33:44,299 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt requirements.txt +2025-01-07 16:33:44,301 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:33:44,301 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:44,303 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log output.log +2025-01-07 16:33:44,306 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:33:44,307 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 media/videos/eval/0_eval_0_2cb3ba1fd3a94289841e.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 media/videos/eval/4_eval_4_7ab8def83cb73c444594.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 media/videos/eval/3_eval_3_2751d2990cf3529f7f75.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 media/videos/eval/1_eval_1_9718732bc54ea0f36a22.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 media/videos/eval/2_eval_2_75fd82618aef0bd510b9.mp4 +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json media/table/eval/total_success_rate_ci_6_3f23270287b36641e4b1.table.json +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 16:33:44,308 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:33:44,308 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,308 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:33:44,308 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,309 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:33:44,309 INFO SenderThread:2184937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:33:44,332 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,332 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:44,471 INFO wandb-upload_0:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/requirements.txt +2025-01-07 16:33:44,498 INFO wandb-upload_1:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/config.yaml +2025-01-07 16:33:44,515 INFO wandb-upload_2:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/wandb-summary.json +2025-01-07 16:33:44,517 INFO wandb-upload_4:2184937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/files/output.log +2025-01-07 16:33:44,583 INFO wandb-upload_3:2184937 [upload_job.py:push():130] Uploaded file /tmp/tmpvw_os9m8wandb/li31m1s1-events.out.tfevents.1736285295.leonmkim-ROG-Strix-G15CS-G15CS.2184761.0 +2025-01-07 16:33:44,783 INFO Thread-11 (_thread_body):2184937 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:33:44,784 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,784 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:33:44,784 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,784 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:33:44,784 INFO SenderThread:2184937 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:33:44,785 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:33:44,785 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,785 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:33:44,786 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,786 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:33:44,786 INFO SenderThread:2184937 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:33:44,973 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,973 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:33:44,973 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:33:44,973 INFO SenderThread:2184937 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:33:44,973 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:33:44,974 INFO HandlerThread:2184937 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send():379] send: final +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send():379] send: footer +2025-01-07 16:33:44,974 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: defer +2025-01-07 16:33:44,974 INFO SenderThread:2184937 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:33:44,974 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,975 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:33:44,975 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:33:44,975 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:44,975 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:33:44,976 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:33:44,977 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:33:44,978 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:33:45,028 DEBUG SenderThread:2184937 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:33:45,028 INFO MainThread:2184937 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:33:45,029 INFO MainThread:2184937 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:33:45,029 INFO MainThread:2184937 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:33:45,029 DEBUG HandlerThread:2184937 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:33:45,029 INFO HandlerThread:2184937 [handler.py:finish():882] shutting down handler +2025-01-07 16:33:45,975 INFO WriterThread:2184937 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [sender.py:finish():1615] shutting down sender +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:33:46,028 INFO SenderThread:2184937 [file_pusher.py:join():175] waiting for file pusher diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug.log b/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5c0c0e12419251b6b5d5a901f0856f2c1b967f31 --- /dev/null +++ b/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Configure stats pid to 2184761 +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/settings +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:28:14,933 INFO MainThread:2184761 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug.log +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/wandb/run-20250107_162814-qgwo1435/logs/debug-internal.log +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1017_0', 'wandb_notes': '1017_0_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435'} +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():619] starting backend +2025-01-07 16:28:14,934 INFO MainThread:2184761 [wandb_init.py:init():623] setting up manager +2025-01-07 16:28:14,938 INFO MainThread:2184761 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:28:14,939 INFO MainThread:2184761 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:28:14,952 INFO MainThread:2184761 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:28:14,959 INFO MainThread:2184761 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:28:15,207 INFO MainThread:2184761 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:28:15,277 INFO MainThread:2184761 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:28:15,278 INFO MainThread:2184761 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:28:15,620 INFO MainThread:2184761 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:28:15,622 INFO MainThread:2184761 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:28:15,623 INFO MainThread:2184761 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1017_0/qgwo1435/tb, True +2025-01-07 16:28:18,361 INFO MainThread:2184761 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:33:46,030 WARNING MsgRouterThr:2184761 [router.py:message_loop():77] message_loop has been closed diff --git a/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb b/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb new file mode 100644 index 0000000000000000000000000000000000000000..84719e1971e65201c2ac92d865e17d9fd84c6a84 Binary files /dev/null and b/qgwo1435/wandb/run-20250107_162814-qgwo1435/run-qgwo1435.wandb differ diff --git a/up4gjipz/episode_rosbags/episode_1_2025-01-06-20-02-04.bag b/up4gjipz/episode_rosbags/episode_1_2025-01-06-20-02-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..a98a0b2bc9983085417d8df78e80502f0b3aa036 --- /dev/null +++ b/up4gjipz/episode_rosbags/episode_1_2025-01-06-20-02-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b91d4d64130e5f472ee3eebefe260e7e70ab77045c3600bed44bf4bc414ba67 +size 2596042053 diff --git a/up4gjipz/episode_rosbags/episode_4_2025-01-06-20-04-42.bag b/up4gjipz/episode_rosbags/episode_4_2025-01-06-20-04-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..17a0af47585a753c730657cb872f9794ebdafa1e --- /dev/null +++ b/up4gjipz/episode_rosbags/episode_4_2025-01-06-20-04-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:894d808f6865a73520f4cd442eff16ec62dd61b8ccb1ba5aaff2e1bd7c07ed0f +size 2599316618 diff --git a/x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag b/x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..a50577ed3db238a39baa171c9f34aecd355cc0e2 --- /dev/null +++ b/x85ya2ao/episode_rosbags/episode_0_2025-01-07-18-28-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab77c29e297a1bb7d56fcb7113553e5baee5c30fcfd430a81abd401a3fcf67f +size 2617717685 diff --git a/x85ya2ao/episode_rosbags/episode_1_2025-01-07-18-29-41.bag b/x85ya2ao/episode_rosbags/episode_1_2025-01-07-18-29-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..4ef2c0ba5f8a4a7d2c25ddb1bb81cbaa56b7774d --- /dev/null +++ b/x85ya2ao/episode_rosbags/episode_1_2025-01-07-18-29-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18fc2b6e5d402344f056fc6090374b2351e3979c277bd0fc2edebdac6cfedd6b +size 2607729670 diff --git a/x85ya2ao/episode_rosbags/episode_2_2025-01-07-18-30-30.bag b/x85ya2ao/episode_rosbags/episode_2_2025-01-07-18-30-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..f6dcdeb9683e0228df716f76be18eb09ad65ea9b --- /dev/null +++ b/x85ya2ao/episode_rosbags/episode_2_2025-01-07-18-30-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c2eb9507a6fde647d73fb2d3b7d0d98c6b81fdf87d54975b2e9becda6b6c97 +size 2603911823 diff --git a/x85ya2ao/episode_rosbags/episode_3_2025-01-07-18-31-27.bag b/x85ya2ao/episode_rosbags/episode_3_2025-01-07-18-31-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..910d2b7f6345d1d15e036b423dbb471a45a639c7 --- /dev/null +++ b/x85ya2ao/episode_rosbags/episode_3_2025-01-07-18-31-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:362bebfa4499311c05fe45197cc8824bb897bb16f25821b362cea124651bf9ce +size 2608933124 diff --git a/x85ya2ao/episode_rosbags/episode_4_2025-01-07-18-32-14.bag b/x85ya2ao/episode_rosbags/episode_4_2025-01-07-18-32-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..8e6b33c9e435f5e5db3f33296c808932270c325c --- /dev/null +++ b/x85ya2ao/episode_rosbags/episode_4_2025-01-07-18-32-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60a8601e163d0269e357fb2e91f1dacfbfaccbffa447197157683267b042dff7 +size 2608235749