diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..0311eae82d7757d4763c74efc1d7d645c57a3895 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,39 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +52k33ku2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +5q1nh8a1/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3cjc38bu/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/episode_rosbags/episode_0_2025-01-03-22-40-23.bag filter=lfs diff=lfs merge=lfs -text +n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 filter=lfs diff=lfs merge=lfs -text +n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3cjc38bu/.hydra/config.yaml b/3cjc38bu/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/3cjc38bu/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/3cjc38bu/.hydra/hydra.yaml b/3cjc38bu/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33036e6a020e7662a71f15248bb62def5ce5d41d --- /dev/null +++ b/3cjc38bu/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/3cjc38bu/.hydra/overrides.yaml b/3cjc38bu/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/3cjc38bu/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/3cjc38bu/episode_rosbags/aligned_depth_to_color_K.npy b/3cjc38bu/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3cjc38bu/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3cjc38bu/episode_rosbags/cam_tf_world.npy b/3cjc38bu/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/3cjc38bu/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/3cjc38bu/episode_rosbags/color_K.npy b/3cjc38bu/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3cjc38bu/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3cjc38bu/episode_rosbags/depth_K.npy b/3cjc38bu/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/3cjc38bu/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/3cjc38bu/eval_robot.log b/3cjc38bu/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..df5d3f308b37b2dc1791ed238c3eb7fb81ae6085 --- /dev/null +++ b/3cjc38bu/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 20:53:13,015][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:53:13,017][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/3cjc38bu/eval_video/0_eval.mp4 b/3cjc38bu/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79ff585d684eb25cbc1b92c975505883476d3587 --- /dev/null +++ b/3cjc38bu/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:096f39e644ca4f96ed5f9dd174de4b96266c79c837e388f8be899d7f0b4d5fed +size 1150163 diff --git a/3cjc38bu/eval_video/1_eval.mp4 b/3cjc38bu/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df15575eaa5740dbe5385faaa3d9556593adbc1d --- /dev/null +++ b/3cjc38bu/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e133f078647a95ae56c93d0f572ed8fe4dbca6c048410bce736a3a298ccf22 +size 1142587 diff --git a/3cjc38bu/eval_video/2_eval.mp4 b/3cjc38bu/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e446355f3f1c7b7f5ec074876261734fda991b41 --- /dev/null +++ b/3cjc38bu/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e0a3706c6d0b154c56d4e16360f718b4b6bc002ed6e7ba2a31af6ae2da4972f +size 1140328 diff --git a/3cjc38bu/eval_video/3_eval.mp4 b/3cjc38bu/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b701a633ea29b6e2ee2ae26dc10b2219170da15e --- /dev/null +++ b/3cjc38bu/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a5c5655a569c9416d7b4ff1810213f1156a81a2a74d6632b54de8738684799b +size 1144801 diff --git a/3cjc38bu/eval_video/4_eval.mp4 b/3cjc38bu/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef23d06339fda8a68a2301b9ee442b8db8a43540 --- /dev/null +++ b/3cjc38bu/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86bb9997ab4f53423525f831efdb1dac4772509c608415678aa65f26e18ae8fc +size 1153082 diff --git a/3cjc38bu/tb/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 b/3cjc38bu/tb/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 new file mode 100644 index 0000000000000000000000000000000000000000..f21ffbaec9b7f004aa4b9a24ea9f821286737ae5 --- /dev/null +++ b/3cjc38bu/tb/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1741cfd5c99f7f02dd53af348cbba96d13b1db26e54a583a641e8d05e0da8359 +size 6539 diff --git a/3cjc38bu/wandb/debug-internal.log b/3cjc38bu/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3a74e915e57a60c23353c6141fd2184b870ed788 --- /dev/null +++ b/3cjc38bu/wandb/debug-internal.log @@ -0,0 +1,747 @@ +2025-01-04 20:53:18,773 INFO StreamThr :2018735 [internal.py:wandb_internal():85] W&B internal server running at pid: 2018735, started at: 2025-01-04 20:53:18.773056 +2025-01-04 20:53:18,775 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:53:18,788 INFO WriterThread:2018735 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb +2025-01-04 20:53:18,788 DEBUG SenderThread:2018735 [sender.py:send():379] send: header +2025-01-04 20:53:18,796 DEBUG SenderThread:2018735 [sender.py:send():379] send: run +2025-01-04 20:53:19,027 INFO SenderThread:2018735 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files +2025-01-04 20:53:19,027 INFO SenderThread:2018735 [sender.py:_start_run_threads():1188] run started: 3cjc38bu with start time 1736041998.774083 +2025-01-04 20:53:19,042 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:53:19,042 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:53:19,099 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:53:19,104 DEBUG HandlerThread:2018735 [system_info.py:__init__():26] System info init +2025-01-04 20:53:19,104 DEBUG HandlerThread:2018735 [system_info.py:__init__():41] System info init done +2025-01-04 20:53:19,104 INFO HandlerThread:2018735 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:53:19,104 INFO SystemMonitor:2018735 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:53:19,104 INFO HandlerThread:2018735 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:53:19,104 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:53:19,105 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:53:19,105 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:53:19,106 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:53:19,106 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:53:19,116 DEBUG HandlerThread:2018735 [system_info.py:probe():152] Probing system +2025-01-04 20:53:19,118 DEBUG HandlerThread:2018735 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:probe():200] Probing system done +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:53:19.116699', 'startedAt': '2025-01-05T01:53:18.760486', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.992375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 630.429874420166}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:53:19,122 INFO HandlerThread:2018735 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:53:19,122 INFO HandlerThread:2018735 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:_save_code():44] Saving code +2025-01-04 20:53:19,130 DEBUG HandlerThread:2018735 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:53:19,130 DEBUG HandlerThread:2018735 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:53:19,191 DEBUG HandlerThread:2018735 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:53:19,192 INFO HandlerThread:2018735 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:53:19,195 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:53:19,195 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:53:19,195 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:53:19,196 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:53:19,430 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/zm5ytv32-wandb-metadata.json +2025-01-04 20:53:19,441 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:53:19,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:53:19,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:19,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:19,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:19,444 INFO HandlerThread:2018735 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/tb" + save: true +} +control { + relay_id: "05d67569aae8429a8a88c401bbd938b9" +} +_info { + stream_id: "3cjc38bu" +} + +2025-01-04 20:53:19,444 DEBUG HandlerThread:2018735 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:53:19,462 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/etbtlxpd-diff.patch +2025-01-04 20:53:19,464 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/kwoc6mul-code/FISH/eval_robot.py +2025-01-04 20:53:19,520 DEBUG SenderThread:2018735 [sender.py:send():379] send: telemetry +2025-01-04 20:53:19,521 DEBUG SenderThread:2018735 [sender.py:send():379] send: tbrecord +2025-01-04 20:53:19,521 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:53:19,521 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 with policy live +2025-01-04 20:53:19,695 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/ophwgxlh-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:53:20,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:20,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code +2025-01-04 20:53:20,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:21,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:22,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:22,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:23,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:24,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:24,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:24,814 DEBUG SenderThread:2018735 [sender.py:send():379] send: config +2025-01-04 20:53:25,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:26,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:26,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:27,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:28,030 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:28,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:29,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:29,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:30,030 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:30,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:31,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:32,031 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:32,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:33,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:34,032 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:34,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:34,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:34,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:34,574 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:35,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:36,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:37,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:38,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:39,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:40,033 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:40,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:40,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:41,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:42,034 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:42,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:43,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:44,034 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:44,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:45,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:46,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:46,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:47,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:48,035 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:48,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:49,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:49,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:49,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:50,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:51,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:51,594 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:52,037 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:53:52,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:53,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:54,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:55,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:56,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:57,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:57,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:58,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:59,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:00,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:01,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:02,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:02,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:03,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:04,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:04,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:04,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:05,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:06,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:07,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:07,597 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:08,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:09,461 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:10,041 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:10,297 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:54:10,297 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 with policy now +2025-01-04 20:54:10,298 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:54:10,298 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:54:10,299 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:54:10,299 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:54:10,461 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:10,515 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/os8v22be-media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media +2025-01-04 20:54:11,310 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/jqa3dx81-media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:12,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:12,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:13,316 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:13,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:14,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:15,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:16,044 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:16,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:17,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:18,044 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:18,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:18,770 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:19,106 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:54:19,107 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:54:19,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:19,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:19,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:20,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:21,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:22,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:23,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:24,253 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:24,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:25,047 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:54:25,309 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/2ie2c098-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:54:25,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:26,047 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:26,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:27,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:28,048 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:28,468 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:29,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:29,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:30,048 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:30,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:31,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:32,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:33,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:34,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:34,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:34,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:34,599 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:35,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:36,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:37,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:38,051 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:38,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:39,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:40,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:40,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:41,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:42,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:43,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:44,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:45,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:45,825 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:46,053 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:46,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:47,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:48,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:49,108 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:54:49,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:49,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:49,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:50,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:51,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:51,592 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:52,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:53,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:54,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:55,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:56,014 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:54:56,014 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 with policy now +2025-01-04 20:54:56,017 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:54:56,017 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:54:56,017 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:54:56,018 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:54:56,235 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/ab6jnmbl-media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:56,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:56,486 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/3b9als3v-media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:57,029 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:57,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:58,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:58,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:59,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:00,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:01,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:02,059 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:02,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:02,509 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:03,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:04,059 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:04,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:04,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:04,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:05,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:06,060 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:06,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:06,850 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:06,851 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:06,851 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:07,061 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:07,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:08,025 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:08,061 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:08,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:09,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:10,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:11,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:12,062 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:12,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:13,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:13,959 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:14,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:15,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:16,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:17,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:18,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:19,108 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:55:19,109 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:19,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:19,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:19,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:20,478 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:21,478 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:22,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:23,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:24,458 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:24,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:25,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:26,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:27,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:28,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:29,458 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:29,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:30,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:31,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:32,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:33,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:34,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:34,444 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:34,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:34,545 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:35,482 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:36,482 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:37,486 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:37,761 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:55:37,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 with policy now +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:37,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:37,987 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/et1qucwe-media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:55:38,324 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/avvudqcw-media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:39,490 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:40,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:40,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:40,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:41,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:42,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:43,497 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:44,073 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:44,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:45,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:46,073 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:46,460 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:46,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:47,492 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:48,493 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:49,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:55:49,448 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:49,448 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:49,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:50,492 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:51,494 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:51,592 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:52,501 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:53,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:54,498 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:55,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:56,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:56,620 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:56,690 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:56,690 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:56,692 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:57,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:57,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:57,309 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/nbjahrnm-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:57,496 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:58,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:58,499 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:59,498 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:00,078 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:00,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:01,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:01,789 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:02,079 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:02,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:03,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:04,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:04,445 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:04,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:05,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:06,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:07,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:07,590 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:08,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:09,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:10,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:11,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:12,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:13,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:13,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:14,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:15,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:16,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:17,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:18,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:19,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:56:19,109 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:19,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:19,445 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:19,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:20,085 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:20,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:21,515 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:22,515 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:23,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:24,087 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:24,135 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:24,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:25,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:26,088 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:26,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:27,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:28,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:28,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:28,599 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:56:28,599 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 with policy now +2025-01-04 20:56:28,601 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:56:28,602 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:56:28,602 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:56:28,603 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:56:28,844 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/mqucimki-media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:56:29,347 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/g41wyz81-media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,470 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:29,525 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:30,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:30,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:31,522 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:32,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:33,527 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:34,091 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:34,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:34,451 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:34,530 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:34,547 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:35,527 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:36,091 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:36,534 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:37,531 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:38,532 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:39,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:40,471 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:40,533 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:41,534 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:42,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:43,543 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:43,760 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:56:43,761 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:56:43,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:56:44,094 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:56:44,095 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:56:44,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:45,539 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:45,749 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:46,095 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:46,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:47,541 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:48,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:49,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:56:49,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:49,447 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:49,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:50,097 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:50,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:51,509 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:51,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:52,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:53,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:54,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:55,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:56,099 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:56,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:57,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:57,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:58,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:59,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:00,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:01,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:02,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:02,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:03,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:04,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:57:04,447 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:57:04,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:05,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:06,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:07,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:07,548 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:08,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:09,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:10,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:11,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:12,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:13,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:13,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:14,539 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:15,547 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:15,942 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:15,942 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 with policy now +2025-01-04 20:57:15,944 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:15,945 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:15,945 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:15,945 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:57:16,158 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/havlzp6u-media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,430 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/pbvimx8s-media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,554 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:17,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:18,107 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:18,572 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:18,961 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:19,110 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:57:19,454 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:57:19,454 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:57:19,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:20,583 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:21,559 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:22,108 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:22,567 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:23,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:24,109 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:24,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:24,564 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:25,572 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:25,809 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:25,809 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:25,810 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:25,884 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:57:25,884 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:57:26,036 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:26,109 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval +2025-01-04 20:57:26,283 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/pdfcpifp-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:26,367 INFO wandb-upload_2:2018735 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwxmmbix0 +2025-01-04 20:57:26,385 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/l84vo94r-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:26,438 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:57:26,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:26,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:26,729 INFO SenderThread:2018735 [sender.py:send_request_log_artifact():1525] logged artifact run-3cjc38bu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyNTYxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTg5MQ==', 'latestArtifact': None}} +2025-01-04 20:57:26,729 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:26,729 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 20:57:26,730 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:26,730 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,730 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,731 DEBUG SenderThread:2018735 [sender.py:send():379] send: telemetry +2025-01-04 20:57:26,731 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,734 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,734 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,735 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,735 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,736 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,736 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,736 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,736 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,737 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,737 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,737 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,737 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,738 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,738 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,739 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,739 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,739 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:57:26,954 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/4no4hkxi-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval +2025-01-04 20:57:27,124 INFO wandb-upload_1:2018735 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4borbvzg +2025-01-04 20:57:27,274 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/tf4dupv5-media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:27,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:27,636 INFO SenderThread:2018735 [sender.py:send_request_log_artifact():1525] logged artifact run-3cjc38bu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyNTYzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTg5Mw==', 'latestArtifact': None}} +2025-01-04 20:57:27,636 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:27,636 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json with policy now +2025-01-04 20:57:27,636 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:27,637 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:27,637 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:27,887 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/uqhq7ggw-media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:28,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:28,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:28,473 DEBUG SenderThread:2018735 [sender.py:send():379] send: exit +2025-01-04 20:57:28,473 INFO SenderThread:2018735 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_exit():588] handling runtime: 249 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_exit():594] send defer +2025-01-04 20:57:28,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,474 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:57:28,474 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:57:28,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:57:28,475 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,475 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:57:28,475 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:57:28,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:57:28,475 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:57:28,475 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:57:28,476 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:57:28,476 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:57:28,527 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:57:28,528 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:57:28,528 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:57:28,528 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,528 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:57:28,528 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:57:28,528 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:57:28,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,529 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:57:28,529 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,529 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:57:28,529 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:57:28,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,529 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:57:29,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:30,112 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,817 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:57:34,817 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:34,817 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,817 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:57:34,818 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:34,818 DEBUG SenderThread:2018735 [sender.py:send():379] send: summary +2025-01-04 20:57:34,821 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:34,822 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:57:34,822 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,822 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:57:34,822 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:57:34,825 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,957 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:57:34,958 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,958 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:57:34,958 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,958 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:57:35,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:57:35,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:35,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:35,783 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:57:35,783 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:35,784 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:35,784 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:57:35,784 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:35,784 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:57:35,784 INFO SenderThread:2018735 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:57:35,786 INFO SenderThread:2018735 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:57:35,788 INFO SenderThread:2018735 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:57:35,803 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:57:35,803 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:35,803 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:57:35,803 DEBUG SenderThread:2018735 [sender.py:send():379] send: artifact +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'versionIndex': 418}}} +2025-01-04 20:57:36,020 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:57:36,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml config.yaml +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json wandb-summary.json +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt requirements.txt +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log output.log +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch diff.patch +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:57:36,117 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:57:36,120 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,120 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:57:36,123 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,124 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:57:36,125 INFO SenderThread:2018735 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:57:36,309 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:36,334 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:57:36,339 INFO wandb-upload_3:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt +2025-01-04 20:57:36,347 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/gctwz2mc-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:36,402 INFO wandb-upload_4:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:36,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,475 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,602 INFO Thread-11 (_thread_body):2018735 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:57:36,602 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,602 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:57:36,602 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,602 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:57:36,603 INFO SenderThread:2018735 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:57:36,604 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,604 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:57:36,604 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:57:36,682 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,682 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:57:36,682 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:57:36,682 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,682 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send():379] send: final +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send():379] send: footer +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,683 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:57:36,683 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,683 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:57:36,684 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,684 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:57:36,685 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:36,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,735 INFO MainThread:2018735 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:57:36,735 INFO MainThread:2018735 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:57:36,736 INFO MainThread:2018735 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:57:36,736 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:57:36,736 INFO HandlerThread:2018735 [handler.py:finish():882] shutting down handler +2025-01-04 20:57:37,686 INFO WriterThread:2018735 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [sender.py:finish():1615] shutting down sender +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [file_pusher.py:join():175] waiting for file pusher diff --git a/3cjc38bu/wandb/debug.log b/3cjc38bu/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..500141ee7fb1b400b3bc73093aba9f1b937f4eac --- /dev/null +++ b/3cjc38bu/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Configure stats pid to 2018568 +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/settings +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug.log +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug-internal.log +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu'} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():619] starting backend +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():623] setting up manager +2025-01-04 20:53:18,772 INFO MainThread:2018568 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:53:18,773 INFO MainThread:2018568 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:53:18,787 INFO MainThread:2018568 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:53:18,794 INFO MainThread:2018568 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:53:19,042 INFO MainThread:2018568 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:53:19,094 INFO MainThread:2018568 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:53:19,094 INFO MainThread:2018568 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:53:19,443 INFO MainThread:2018568 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:53:19,444 INFO MainThread:2018568 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/tb, True +2025-01-04 20:53:24,813 INFO MainThread:2018568 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:57:37,736 WARNING MsgRouterThr:2018568 [router.py:message_loop():77] message_loop has been closed diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd242acb28ecdfc4039a5cec58ad30e8cceb3ffc --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736041998 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..404b5ba8a95c9a7b97eb1e7507c090d49a2961c8 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[38, 40, 0.95, 0.8149728178348021, 0.9950384329014377, 0.8521535544883234, 0.99369423547199, 0.870794389322747, 0.9856821223122073, 0.8775677233179267, 0.9856885344681322]]} \ No newline at end of file diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79ff585d684eb25cbc1b92c975505883476d3587 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:096f39e644ca4f96ed5f9dd174de4b96266c79c837e388f8be899d7f0b4d5fed +size 1150163 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df15575eaa5740dbe5385faaa3d9556593adbc1d --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e133f078647a95ae56c93d0f572ed8fe4dbca6c048410bce736a3a298ccf22 +size 1142587 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e446355f3f1c7b7f5ec074876261734fda991b41 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e0a3706c6d0b154c56d4e16360f718b4b6bc002ed6e7ba2a31af6ae2da4972f +size 1140328 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b701a633ea29b6e2ee2ae26dc10b2219170da15e --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a5c5655a569c9416d7b4ff1810213f1156a81a2a74d6632b54de8738684799b +size 1144801 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef23d06339fda8a68a2301b9ee442b8db8a43540 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86bb9997ab4f53423525f831efdb1dac4772509c608415678aa65f26e18ae8fc +size 1153082 diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2a366883ececc3be820b22acc70ef10e1732e87c --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736042004.814549]: resetting environment +[INFO] [1736042004.819769]: cleared current plan +[INFO] [1736042004.820590]: moving to home +[INFO] [1736042005.822918]: reached home +[INFO] [1736042005.824436]: reset action history +[INFO] [1736042008.532802]: environment reset +Starting episode 0 +[INFO] [1736042008.534993]: resetting environment +[INFO] [1736042008.537605]: cleared current plan +[INFO] [1736042008.538408]: moving to home +[INFO] [1736042009.540878]: reached home +[INFO] [1736042009.542279]: reset action history +[INFO] [1736042012.249875]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736042018.509835]: resetting environment +[INFO] [1736042018.512872]: cleared current plan +[INFO] [1736042018.514173]: moving to home +[INFO] [1736042019.516171]: reached home +[INFO] [1736042019.518075]: reset action history +[INFO] [1736042022.229890]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/episode_rosbags/episode_0_2025-01-04-20-53-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2018568.log +started roslaunch server http://158.130.50.37:45309/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2018846] +process[print_text-2]: started with pid [2018870] +process[rosbag_record-3]: started with pid [2018871] +started bagging! +For topic gripper_width: timestamp difference is -121691404 for nearest: 1736042026251565518 - target: 1736042026373256922 at idx 53 +For topic gripper_width: timestamp difference is 101847974 for nearest: 1736042048118232576 - target: 1736042048016384602 at idx 249 +[INFO] [1736042049.287461]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736042050.079745]: Stored episode 1. +[INFO] [1736042050.080131]: Saving video... +[INFO] [1736042050.281159]: Video saved! +Starting episode 1 +[INFO] [1736042050.299864]: resetting environment +[INFO] [1736042050.314869]: cleared current plan +[INFO] [1736042050.315585]: moving to home +[INFO] [1736042054.017320]: reached home +[INFO] [1736042054.019873]: reset action history +[INFO] [1736042056.726394]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042064.257640]: resetting environment +[INFO] [1736042064.259552]: cleared current plan +[INFO] [1736042064.259701]: moving to home +[INFO] [1736042065.261509]: reached home +[INFO] [1736042065.262238]: reset action history +[INFO] [1736042067.969493]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/episode_rosbags/episode_1_2025-01-04-20-54-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2018568.log +started roslaunch server http://158.130.50.37:34321/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2018980] +process[print_text-5]: started with pid [2018981] +process[rosbag_record-6]: started with pid [2018982] +started bagging! +For topic gripper_width: timestamp difference is -126093135 for nearest: 1736042076551565184 - target: 1736042076677658319 at idx 108 +For topic gripper_width: timestamp difference is -243139308 for nearest: 1736042084451605994 - target: 1736042084694745302 at idx 208 +For topic gripper_width: timestamp difference is -100305063 for nearest: 1736042093951588410 - target: 1736042094051893473 at idx 249 +[INFO] [1736042094.935066]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736042095.700144]: Stored episode 2. +[INFO] [1736042095.700997]: Saving video... +[INFO] [1736042095.995893]: Video saved! +Starting episode 2 +[INFO] [1736042096.018546]: resetting environment +[INFO] [1736042096.028081]: cleared current plan +[INFO] [1736042096.028354]: moving to home +[INFO] [1736042099.735414]: reached home +[INFO] [1736042099.741363]: reset action history +[INFO] [1736042102.464514]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042105.978211]: resetting environment +[INFO] [1736042105.982179]: cleared current plan +[INFO] [1736042105.982356]: moving to home +[INFO] [1736042106.983948]: reached home +[INFO] [1736042106.984971]: reset action history +[INFO] [1736042109.699395]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/episode_rosbags/episode_2_2025-01-04-20-55-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2018568.log +started roslaunch server http://158.130.50.37:37299/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2019126] +process[print_text-8]: started with pid [2019127] +process[rosbag_record-9]: started with pid [2019151] +started bagging! +[INFO] [1736042136.658378]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736042137.450518]: Stored episode 3. +[INFO] [1736042137.452666]: Saving video... +[INFO] [1736042137.716969]: Video saved! +Starting episode 3 +[INFO] [1736042137.750825]: resetting environment +[INFO] [1736042137.763510]: cleared current plan +[INFO] [1736042137.766447]: moving to home +[INFO] [1736042141.469090]: reached home +[INFO] [1736042141.473977]: reset action history +[INFO] [1736042144.263789]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042156.633212]: resetting environment +[INFO] [1736042156.635866]: cleared current plan +[INFO] [1736042156.636067]: moving to home +[INFO] [1736042157.637291]: reached home +[INFO] [1736042157.638444]: reset action history +[INFO] [1736042160.361292]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/episode_rosbags/episode_3_2025-01-04-20-56-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2018568.log +started roslaunch server http://158.130.50.37:40783/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2019275] +process[print_text-11]: started with pid [2019276] +process[rosbag_record-12]: started with pid [2019300] +started bagging! +For topic gripper_width: timestamp difference is 100920159 for nearest: 1736042178118257958 - target: 1736042178017337799 at idx 242 +For topic gripper_width: timestamp difference is 104304858 for nearest: 1736042182351530143 - target: 1736042182247225285 at idx 249 +For topic gripper_width: timestamp difference is 105728456 for nearest: 1736042184118233272 - target: 1736042184012504816 at idx 249 +For topic gripper_width: timestamp difference is 107518943 for nearest: 1736042186318229230 - target: 1736042186210710287 at idx 249 +For topic gripper_width: timestamp difference is 109634293 for nearest: 1736042186520179642 - target: 1736042186410545349 at idx 249 +[INFO] [1736042187.510129]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736042188.279175]: Stored episode 4. +[INFO] [1736042188.285867]: Saving video... +[INFO] [1736042188.556105]: Video saved! +Starting episode 4 +[INFO] [1736042188.603145]: resetting environment +[INFO] [1736042188.626212]: cleared current plan +[INFO] [1736042188.630020]: moving to home +[INFO] [1736042192.333293]: reached home +[INFO] [1736042192.335803]: reset action history +[INFO] [1736042195.106601]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736042203.691004]: resetting environment +[INFO] [1736042203.705134]: cleared current plan +[INFO] [1736042203.705665]: moving to home +[INFO] [1736042204.707591]: reached home +[INFO] [1736042204.713738]: reset action history +[INFO] [1736042207.494369]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/episode_rosbags/episode_4_2025-01-04-20-56-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2018568.log +started roslaunch server http://158.130.50.37:37667/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2019444] +process[print_text-14]: started with pid [2019445] +process[rosbag_record-15]: started with pid [2019446] +started bagging! +For topic gripper_width: timestamp difference is -103699979 for nearest: 1736042213451587144 - target: 1736042213555287123 at idx 87 +[INFO] [1736042234.616855]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736042235.392855]: Stored episode 5. +[INFO] [1736042235.394972]: Saving video... +[INFO] [1736042235.891070]: Video saved! +[INFO] [1736042235.946117]: resetting environment +[INFO] [1736042235.958890]: cleared current plan +[INFO] [1736042235.960421]: moving to home +[INFO] [1736042239.668089]: reached home +[INFO] [1736042239.675882]: reset action history +[INFO] [1736042242.521391]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..048a5c8a03553c4688c0346a24ff7fe41fc667f2 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T01:53:19.116699", + "startedAt": "2025-01-05T01:53:18.760486", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.992375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.024, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 630.429874420166 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..038d88eeaa0aa9f32aa9a6f692b41082de5c0989 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "096f39e644ca4f96ed5f9dd174de4b96266c79c837e388f8be899d7f0b4d5fed", "size": 1150163, "path": "media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4"}, "global_step": 664, "_timestamp": 1736042246.3558095, "_runtime": 247.5817265510559, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b6e133f078647a95ae56c93d0f572ed8fe4dbca6c048410bce736a3a298ccf22", "size": 1142587, "path": "media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4"}, "eval/num_success": 5.0, "episode": 40.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "6e0a3706c6d0b154c56d4e16360f718b4b6bc002ed6e7ba2a31af6ae2da4972f", "size": 1140328, "path": "media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2a5c5655a569c9416d7b4ff1810213f1156a81a2a74d6632b54de8738684799b", "size": 1144801, "path": "media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "86bb9997ab4f53423525f831efdb1dac4772509c608415678aa65f26e18ae8fc", "size": 1153082, "path": "media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://hvvgctzjq1c1deec2cm8nynyw64ggmuggzrrkmr1nkg9t4ij4yznbpgoqsn2m94gw0lkg27epltpsx7b1ukzos8pbeaszrqxm8o2g4l5zt9bpcsb3m27dy690dhs8rpx/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zj5eqoo5kirpubbabzsnjtju54pkct9kzrr5gusqoay71fpw6a0iwuf6c868ecvsorw51rg87484ubq094unnh8zfpas41rhz31z7quul8h05tws0e0hblqtycttwe5w:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 38, "total_num_episodes": 40, "total_success_rate": 0.95, "total_success_rate_lb_0.01": 0.8149728178348021, "total_success_rate_ub_0.01": 0.9950384329014377, "total_success_rate_lb_0.025": 0.8521535544883234, "total_success_rate_ub_0.025": 0.99369423547199, "total_success_rate_lb_0.05": 0.870794389322747, "total_success_rate_ub_0.05": 0.9856821223122073, "total_success_rate_lb_0.1": 0.8775677233179267, "total_success_rate_ub_0.1": 0.9856885344681322, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "9b18ad5991cde938101ec52ccbd8d8ce6c7dc511bf7eae2fb3a5717568bc9899", "size": 503, "artifact_path": "wandb-client-artifact://xh4kxaf73lkmy1w9c3oyzn8ah2gwr5d0rf5t4vu5pt5undw28uwihv8c3x9ymbkoisxwtt8e67on4jlakumpoxucvhijnx3ln8ihj505yc01iytg8mqrx20cnaqgbi1y/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9yybrw2qqc5n1irgyntcsmyrb3oserzafhqk1l3gxmrvzgwz7bdw89h2ccasgdwa2ejp7mvhawo70w5tjr8cum0q80qw8dzw38lpxpyco4oyx3uif7x3qi74p43405iv:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 249}, "eval/total_success_rate": 0.949999988079071, "eval/total_success_rate_lb_0.01": 0.8149728178977966, "eval/total_success_rate_ub_0.01": 0.9950384497642517, "eval/total_success_rate_lb_0.025": 0.8521535396575928, "eval/total_success_rate_ub_0.025": 0.9936942458152771, "eval/total_success_rate_lb_0.05": 0.870794415473938, "eval/total_success_rate_ub_0.05": 0.9856821298599243, "eval/total_success_rate_lb_0.1": 0.877567708492279, "eval/total_success_rate_ub_0.1": 0.9856885075569153} \ No newline at end of file diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug-internal.log b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3a74e915e57a60c23353c6141fd2184b870ed788 --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug-internal.log @@ -0,0 +1,747 @@ +2025-01-04 20:53:18,773 INFO StreamThr :2018735 [internal.py:wandb_internal():85] W&B internal server running at pid: 2018735, started at: 2025-01-04 20:53:18.773056 +2025-01-04 20:53:18,775 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status +2025-01-04 20:53:18,788 INFO WriterThread:2018735 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb +2025-01-04 20:53:18,788 DEBUG SenderThread:2018735 [sender.py:send():379] send: header +2025-01-04 20:53:18,796 DEBUG SenderThread:2018735 [sender.py:send():379] send: run +2025-01-04 20:53:19,027 INFO SenderThread:2018735 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files +2025-01-04 20:53:19,027 INFO SenderThread:2018735 [sender.py:_start_run_threads():1188] run started: 3cjc38bu with start time 1736041998.774083 +2025-01-04 20:53:19,042 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 20:53:19,042 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: check_version +2025-01-04 20:53:19,099 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 20:53:19,104 DEBUG HandlerThread:2018735 [system_info.py:__init__():26] System info init +2025-01-04 20:53:19,104 DEBUG HandlerThread:2018735 [system_info.py:__init__():41] System info init done +2025-01-04 20:53:19,104 INFO HandlerThread:2018735 [system_monitor.py:start():194] Starting system monitor +2025-01-04 20:53:19,104 INFO SystemMonitor:2018735 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 20:53:19,104 INFO HandlerThread:2018735 [system_monitor.py:probe():214] Collecting system info +2025-01-04 20:53:19,104 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 20:53:19,105 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started disk monitoring +2025-01-04 20:53:19,105 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 20:53:19,106 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started memory monitoring +2025-01-04 20:53:19,106 INFO SystemMonitor:2018735 [interfaces.py:start():188] Started network monitoring +2025-01-04 20:53:19,116 DEBUG HandlerThread:2018735 [system_info.py:probe():152] Probing system +2025-01-04 20:53:19,118 DEBUG HandlerThread:2018735 [system_info.py:_probe_git():137] Probing git +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:_probe_git():145] Probing git done +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:probe():200] Probing system done +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T01:53:19.116699', 'startedAt': '2025-01-05T01:53:18.760486', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.992375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.024, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 630.429874420166}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 20:53:19,122 INFO HandlerThread:2018735 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 20:53:19,122 INFO HandlerThread:2018735 [system_monitor.py:probe():227] Publishing system info +2025-01-04 20:53:19,122 DEBUG HandlerThread:2018735 [system_info.py:_save_code():44] Saving code +2025-01-04 20:53:19,130 DEBUG HandlerThread:2018735 [system_info.py:_save_code():67] Saving code done +2025-01-04 20:53:19,130 DEBUG HandlerThread:2018735 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 20:53:19,191 DEBUG HandlerThread:2018735 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 20:53:19,192 INFO HandlerThread:2018735 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 20:53:19,195 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:53:19,195 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 20:53:19,195 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 20:53:19,196 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 20:53:19,430 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/zm5ytv32-wandb-metadata.json +2025-01-04 20:53:19,441 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 20:53:19,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: python_packages +2025-01-04 20:53:19,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:19,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:19,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:19,444 INFO HandlerThread:2018735 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/tb" + save: true +} +control { + relay_id: "05d67569aae8429a8a88c401bbd938b9" +} +_info { + stream_id: "3cjc38bu" +} + +2025-01-04 20:53:19,444 DEBUG HandlerThread:2018735 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 20:53:19,462 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/etbtlxpd-diff.patch +2025-01-04 20:53:19,464 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/kwoc6mul-code/FISH/eval_robot.py +2025-01-04 20:53:19,520 DEBUG SenderThread:2018735 [sender.py:send():379] send: telemetry +2025-01-04 20:53:19,521 DEBUG SenderThread:2018735 [sender.py:send():379] send: tbrecord +2025-01-04 20:53:19,521 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:53:19,521 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 with policy live +2025-01-04 20:53:19,695 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/ophwgxlh-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:53:20,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:20,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH +2025-01-04 20:53:20,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code +2025-01-04 20:53:20,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:21,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:22,028 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:22,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:23,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:24,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:24,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:24,814 DEBUG SenderThread:2018735 [sender.py:send():379] send: config +2025-01-04 20:53:25,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:26,029 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:26,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:27,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:28,030 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:28,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:29,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:29,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:30,030 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:30,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:31,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:32,031 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:32,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:33,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:34,032 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:34,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:34,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:34,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:34,574 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:35,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:36,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:37,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:38,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:39,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:40,033 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:40,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:40,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:41,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:42,034 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:42,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:43,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:44,034 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:44,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:45,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:46,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:46,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:47,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:48,035 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:53:48,450 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:49,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:53:49,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:53:49,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:50,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:51,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:51,594 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:52,037 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:53:52,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:53,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:54,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:55,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:56,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:57,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:53:57,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:58,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:53:59,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:00,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:01,452 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:02,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:02,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:03,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:04,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:04,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:04,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:05,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:06,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:07,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:07,597 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:08,453 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:09,461 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:10,041 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:10,297 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:54:10,297 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 with policy now +2025-01-04 20:54:10,298 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:54:10,298 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:54:10,299 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:54:10,299 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:54:10,461 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:10,515 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/os8v22be-media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:54:11,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media +2025-01-04 20:54:11,310 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/jqa3dx81-media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:54:11,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:12,042 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:12,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:13,316 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:13,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:14,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:15,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:16,044 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:16,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:17,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:18,044 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:18,463 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:18,770 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:19,106 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 20:54:19,107 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:54:19,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:19,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:19,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:20,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:21,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:22,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:23,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:24,253 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:24,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:25,047 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:54:25,309 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/2ie2c098-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:54:25,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:26,047 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:26,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:27,464 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:28,048 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:28,468 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:29,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:29,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:30,048 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:30,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:31,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:32,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:33,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:34,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:34,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:34,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:34,599 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:35,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:36,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:37,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:38,051 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:38,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:39,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:40,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:40,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:41,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:42,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:43,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:44,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:45,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:45,825 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:46,053 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:46,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:47,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:48,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:49,108 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:54:49,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:54:49,442 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:54:49,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:50,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:51,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:51,592 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:52,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:53,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:54,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:55,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:56,014 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:54:56,014 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 with policy now +2025-01-04 20:54:56,017 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:54:56,017 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:54:56,017 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:54:56,018 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:56,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:54:56,235 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/ab6jnmbl-media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:56,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:56,486 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/3b9als3v-media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:54:57,029 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:54:57,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:58,057 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:54:58,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:54:59,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:00,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:01,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:02,059 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:02,465 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:02,509 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:03,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:04,059 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:04,442 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:04,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:04,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:05,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:06,060 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:06,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:06,850 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:06,851 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:06,851 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:07,061 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:07,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:08,025 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:08,061 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:08,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:09,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:10,476 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:11,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:12,062 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:12,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:13,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:13,959 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:14,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:15,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:16,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:17,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:18,477 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:19,108 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:55:19,109 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:19,443 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:19,443 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:19,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:20,478 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:21,478 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:22,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:23,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:24,458 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:24,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:25,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:26,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:27,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:28,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:29,458 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:29,480 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:30,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:31,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:32,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:33,481 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:34,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:34,444 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:34,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:34,545 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:35,482 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:36,482 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:37,486 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:37,761 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:55:37,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 with policy now +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:37,762 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:37,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:37,987 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/et1qucwe-media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:55:38,324 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/avvudqcw-media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:55:38,485 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:39,490 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:40,071 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:40,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:40,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:41,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:42,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:43,497 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:44,073 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:44,489 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:45,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:46,073 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:46,460 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:46,491 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:47,492 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:48,493 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:49,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:55:49,448 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:55:49,448 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:55:49,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:50,492 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:51,494 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:51,592 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:52,501 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:53,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:54,498 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:55,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:56,495 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:56,620 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:55:56,690 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:55:56,690 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:55:56,692 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:55:57,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:57,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:55:57,309 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/nbjahrnm-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:55:57,496 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:58,077 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:55:58,499 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:55:59,498 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:00,078 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:00,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:01,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:01,789 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:02,079 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:02,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:03,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:04,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:04,445 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:04,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:05,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:06,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:07,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:07,590 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:08,512 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:09,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:10,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:11,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:12,513 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:13,462 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:13,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:14,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:15,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:16,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:17,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:18,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:19,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:56:19,109 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:19,444 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:19,445 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:19,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:20,085 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:20,514 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:21,515 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:22,515 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:23,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:24,087 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:24,135 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:24,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:25,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:26,088 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:26,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:27,516 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:28,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:28,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:28,599 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:56:28,599 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 with policy now +2025-01-04 20:56:28,601 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:56:28,602 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:56:28,602 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:56:28,603 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:56:28,844 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/mqucimki-media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:56:29,347 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/g41wyz81-media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:56:29,470 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:29,525 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:30,089 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:30,521 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:31,522 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:32,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:33,527 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:34,091 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:34,451 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:34,451 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:34,530 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:34,547 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:35,527 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:36,091 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:36,534 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:37,531 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:38,532 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:39,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:40,471 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:40,533 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:41,534 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:42,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:43,543 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:43,760 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:56:43,761 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:56:43,762 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:56:44,094 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:56:44,095 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:56:44,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:45,539 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:45,749 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:46,095 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:46,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:47,541 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:48,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:49,109 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:56:49,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:56:49,447 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:56:49,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:50,097 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:50,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:51,509 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:51,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:52,535 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:53,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:54,536 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:55,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:56,099 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:56:56,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:57,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:56:57,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:58,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:56:59,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:00,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:01,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:02,466 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:02,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:03,537 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:04,446 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:57:04,447 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:57:04,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:05,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:06,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:07,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:07,548 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:08,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:09,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:10,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:11,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:12,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:13,467 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:13,538 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:14,539 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:15,547 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:15,942 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:15,942 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 with policy now +2025-01-04 20:57:15,944 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:15,945 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:15,945 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:15,945 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,106 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval +2025-01-04 20:57:16,158 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/havlzp6u-media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,430 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/pbvimx8s-media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:16,554 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:17,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:18,107 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:18,572 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:18,961 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:19,110 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:57:19,454 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 20:57:19,454 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: stop_status +2025-01-04 20:57:19,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:20,583 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:21,559 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:22,108 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:22,567 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:23,557 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:24,109 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:24,479 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:24,564 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:25,572 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:25,809 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:25,809 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:25,810 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:25,884 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:57:25,884 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:57:26,036 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:26,109 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table +2025-01-04 20:57:26,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval +2025-01-04 20:57:26,283 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/pdfcpifp-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:26,367 INFO wandb-upload_2:2018735 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwxmmbix0 +2025-01-04 20:57:26,385 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/l84vo94r-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:26,438 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 20:57:26,445 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 20:57:26,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:26,729 INFO SenderThread:2018735 [sender.py:send_request_log_artifact():1525] logged artifact run-3cjc38bu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyNTYxNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTg5MQ==', 'latestArtifact': None}} +2025-01-04 20:57:26,729 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:26,729 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 20:57:26,730 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:26,730 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,730 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,731 DEBUG SenderThread:2018735 [sender.py:send():379] send: telemetry +2025-01-04 20:57:26,731 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,734 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,734 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,735 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,735 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,736 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,736 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,736 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,736 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,737 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,737 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,737 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,737 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,738 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,738 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,738 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,739 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:26,739 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:26,739 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 20:57:26,954 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/4no4hkxi-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:27,110 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval +2025-01-04 20:57:27,124 INFO wandb-upload_1:2018735 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4borbvzg +2025-01-04 20:57:27,274 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/tf4dupv5-media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:27,558 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:27,636 INFO SenderThread:2018735 [sender.py:send_request_log_artifact():1525] logged artifact run-3cjc38bu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTEyNTYzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1MTg5Mw==', 'latestArtifact': None}} +2025-01-04 20:57:27,636 DEBUG SenderThread:2018735 [sender.py:send():379] send: files +2025-01-04 20:57:27,636 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json with policy now +2025-01-04 20:57:27,636 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:27,637 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:27,637 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:27,887 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/uqhq7ggw-media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:28,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:28,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:28,473 DEBUG SenderThread:2018735 [sender.py:send():379] send: exit +2025-01-04 20:57:28,473 INFO SenderThread:2018735 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_exit():588] handling runtime: 249 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_exit():594] send defer +2025-01-04 20:57:28,474 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,474 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 20:57:28,474 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 20:57:28,474 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 1 +2025-01-04 20:57:28,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 20:57:28,475 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,475 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 20:57:28,475 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 2 +2025-01-04 20:57:28,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 20:57:28,475 INFO HandlerThread:2018735 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 20:57:28,475 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 20:57:28,475 DEBUG SystemMonitor:2018735 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 20:57:28,476 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 20:57:28,476 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 20:57:28,527 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 20:57:28,528 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 20:57:28,528 INFO HandlerThread:2018735 [interfaces.py:finish():200] Joined network monitor +2025-01-04 20:57:28,528 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,528 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 20:57:28,528 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 3 +2025-01-04 20:57:28,528 DEBUG SenderThread:2018735 [sender.py:send():379] send: stats +2025-01-04 20:57:28,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,529 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 20:57:28,529 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:28,529 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 20:57:28,529 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 4 +2025-01-04 20:57:28,529 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:28,529 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 20:57:29,111 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:30,112 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 20:57:34,815 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,817 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 20:57:34,817 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 5 +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send():379] send: history +2025-01-04 20:57:34,817 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: summary_record +2025-01-04 20:57:34,817 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,817 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 20:57:34,818 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:34,818 DEBUG SenderThread:2018735 [sender.py:send():379] send: summary +2025-01-04 20:57:34,821 INFO SenderThread:2018735 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 20:57:34,822 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 6 +2025-01-04 20:57:34,822 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,822 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 20:57:34,822 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,822 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 20:57:34,825 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 20:57:34,957 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 7 +2025-01-04 20:57:34,958 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:34,958 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 20:57:34,958 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:34,958 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 20:57:35,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:57:35,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:35,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:35,783 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 8 +2025-01-04 20:57:35,783 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:35,784 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:35,784 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 20:57:35,784 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:35,784 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 20:57:35,784 INFO SenderThread:2018735 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 20:57:35,786 INFO SenderThread:2018735 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 20:57:35,788 INFO SenderThread:2018735 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 20:57:35,803 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 9 +2025-01-04 20:57:35,803 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:35,803 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 20:57:35,803 DEBUG SenderThread:2018735 [sender.py:send():379] send: artifact +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'versionIndex': 418}}} +2025-01-04 20:57:36,020 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 20:57:36,020 INFO SenderThread:2018735 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 20:57:36,114 INFO Thread-12 :2018735 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml config.yaml +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json wandb-summary.json +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt requirements.txt +2025-01-04 20:57:36,115 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-metadata.json wandb-metadata.json +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log output.log +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/diff.patch diff.patch +2025-01-04 20:57:36,116 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 20:57:36,117 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 media/videos/eval/4_eval_4_86bb9997ab4f53423525.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 media/videos/eval/2_eval_2_6e0a3706c6d0b154c56d.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 media/videos/eval/0_eval_0_096f39e644ca4f96ed5f.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 media/videos/eval/3_eval_3_2a5c5655a569c9416d7b.mp4 +2025-01-04 20:57:36,118 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 media/videos/eval/1_eval_1_b6e133f078647a95ae56.mp4 +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json media/table/eval/total_success_rate_ci_6_9b18ad5991cde938101e.table.json +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 20:57:36,120 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 10 +2025-01-04 20:57:36,120 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,120 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 20:57:36,123 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,124 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 20:57:36,125 INFO SenderThread:2018735 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:57:36,309 INFO wandb-upload_0:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/wandb-summary.json +2025-01-04 20:57:36,334 INFO wandb-upload_2:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/config.yaml +2025-01-04 20:57:36,339 INFO wandb-upload_3:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/requirements.txt +2025-01-04 20:57:36,347 INFO wandb-upload_1:2018735 [upload_job.py:push():130] Uploaded file /tmp/tmpo1e8gkulwandb/gctwz2mc-events.out.tfevents.1736041999.leonmkim-ROG-Strix-G15CS-G15CS.2018568.0 +2025-01-04 20:57:36,402 INFO wandb-upload_4:2018735 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/files/output.log +2025-01-04 20:57:36,475 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,475 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,602 INFO Thread-11 (_thread_body):2018735 [sender.py:transition_state():614] send defer: 11 +2025-01-04 20:57:36,602 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,602 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 20:57:36,602 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,602 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 20:57:36,603 INFO SenderThread:2018735 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 12 +2025-01-04 20:57:36,604 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,604 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 20:57:36,604 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 20:57:36,604 INFO SenderThread:2018735 [file_stream.py:finish():601] file stream finish called +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [file_stream.py:finish():605] file stream finish is done +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 13 +2025-01-04 20:57:36,682 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,682 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 20:57:36,682 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 20:57:36,682 INFO SenderThread:2018735 [sender.py:transition_state():614] send defer: 14 +2025-01-04 20:57:36,682 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: defer +2025-01-04 20:57:36,682 INFO HandlerThread:2018735 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send():379] send: final +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send():379] send: footer +2025-01-04 20:57:36,683 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: defer +2025-01-04 20:57:36,683 INFO SenderThread:2018735 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 20:57:36,683 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,683 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 20:57:36,684 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,684 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: server_info +2025-01-04 20:57:36,685 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 20:57:36,686 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 20:57:36,735 DEBUG SenderThread:2018735 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 20:57:36,735 INFO MainThread:2018735 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 20:57:36,735 INFO MainThread:2018735 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 20:57:36,736 INFO MainThread:2018735 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 20:57:36,736 DEBUG HandlerThread:2018735 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 20:57:36,736 INFO HandlerThread:2018735 [handler.py:finish():882] shutting down handler +2025-01-04 20:57:37,686 INFO WriterThread:2018735 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [sender.py:finish():1615] shutting down sender +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 20:57:37,735 INFO SenderThread:2018735 [file_pusher.py:join():175] waiting for file pusher diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug.log b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..500141ee7fb1b400b3bc73093aba9f1b937f4eac --- /dev/null +++ b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Configure stats pid to 2018568 +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/settings +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug.log +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/logs/debug-internal.log +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():569] calling init triggers +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu'} +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():619] starting backend +2025-01-04 20:53:18,767 INFO MainThread:2018568 [wandb_init.py:init():623] setting up manager +2025-01-04 20:53:18,772 INFO MainThread:2018568 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 20:53:18,773 INFO MainThread:2018568 [wandb_init.py:init():631] backend started and connected +2025-01-04 20:53:18,787 INFO MainThread:2018568 [wandb_init.py:init():720] updated telemetry +2025-01-04 20:53:18,794 INFO MainThread:2018568 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 20:53:19,042 INFO MainThread:2018568 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 20:53:19,094 INFO MainThread:2018568 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 20:53:19,094 INFO MainThread:2018568 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 20:53:19,442 INFO MainThread:2018568 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 20:53:19,443 INFO MainThread:2018568 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 20:53:19,444 INFO MainThread:2018568 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/3cjc38bu/tb, True +2025-01-04 20:53:24,813 INFO MainThread:2018568 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-04 20:57:37,736 WARNING MsgRouterThr:2018568 [router.py:message_loop():77] message_loop has been closed diff --git a/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7332c0a1ec80d9b1ee2fa6159343032cd77ebfa5 Binary files /dev/null and b/3cjc38bu/wandb/run-20250104_205318-3cjc38bu/run-3cjc38bu.wandb differ diff --git a/52k33ku2/.hydra/config.yaml b/52k33ku2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/52k33ku2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/52k33ku2/.hydra/hydra.yaml b/52k33ku2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43a1f1e61aaf0bf5a910f97445c558fc4dfd0893 --- /dev/null +++ b/52k33ku2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/52k33ku2/.hydra/overrides.yaml b/52k33ku2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/52k33ku2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/52k33ku2/episode_rosbags/aligned_depth_to_color_K.npy b/52k33ku2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/52k33ku2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/52k33ku2/episode_rosbags/cam_tf_world.npy b/52k33ku2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/52k33ku2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/52k33ku2/episode_rosbags/color_K.npy b/52k33ku2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/52k33ku2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/52k33ku2/episode_rosbags/depth_K.npy b/52k33ku2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/52k33ku2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/52k33ku2/eval_robot.log b/52k33ku2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..d35801d201421969453dfd3947994c72638eb989 --- /dev/null +++ b/52k33ku2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:45:17,963][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:45:17,966][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/52k33ku2/eval_video/0_eval.mp4 b/52k33ku2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ef65e5ffba24232635d0795e24d17bf755eab74 --- /dev/null +++ b/52k33ku2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8807ddb3aac3464696fe7106a8211d9caded6db0cac253b390c8fcc9fdc634d +size 1177467 diff --git a/52k33ku2/eval_video/1_eval.mp4 b/52k33ku2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52eca207534b226ed7fa935c2a9b685fb7c19426 --- /dev/null +++ b/52k33ku2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93cac9f740e1662671287f68c5f597c67c4a08fb7e4596ce529b3b96a384d87d +size 1188790 diff --git a/52k33ku2/eval_video/2_eval.mp4 b/52k33ku2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31cb23910e01582f8f1d042ee3ce129b4ecf98c6 --- /dev/null +++ b/52k33ku2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0f4e50e7a3af14dc5d1803d57a0a191579edbd167d55c1b6e7fd9482ff42bb +size 1182853 diff --git a/52k33ku2/eval_video/3_eval.mp4 b/52k33ku2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ede54b0bee14ae59d494a4dd89e97235b2e019a --- /dev/null +++ b/52k33ku2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c01cf859d0173b75d00edc1a7130eb2200d07ee87d70b56f312948359535ac32 +size 1182945 diff --git a/52k33ku2/eval_video/4_eval.mp4 b/52k33ku2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7739eb27093838a8ffa96e2c31e7502966ec1195 --- /dev/null +++ b/52k33ku2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa04d437225714191e3a167bc5aed343b7e8db97f3a29339b2969c15d457517 +size 1182202 diff --git a/52k33ku2/tb/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 b/52k33ku2/tb/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 new file mode 100644 index 0000000000000000000000000000000000000000..b2f213d5a5f6ce1ee50085643e01916996b3f463 --- /dev/null +++ b/52k33ku2/tb/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:118d92645c5022972c5a9f7f74374eb7cde703ef2acbe992f93914aee2a73d60 +size 6539 diff --git a/52k33ku2/wandb/debug-internal.log b/52k33ku2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8b1f90d2f40b82cbf97a1c25dcc97d5a5a942ed4 --- /dev/null +++ b/52k33ku2/wandb/debug-internal.log @@ -0,0 +1,921 @@ +2025-01-03 22:45:23,762 INFO StreamThr :1987335 [internal.py:wandb_internal():85] W&B internal server running at pid: 1987335, started at: 2025-01-03 22:45:23.762311 +2025-01-03 22:45:23,763 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:45:23,774 INFO WriterThread:1987335 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb +2025-01-03 22:45:23,775 DEBUG SenderThread:1987335 [sender.py:send():379] send: header +2025-01-03 22:45:23,785 DEBUG SenderThread:1987335 [sender.py:send():379] send: run +2025-01-03 22:45:24,051 INFO SenderThread:1987335 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files +2025-01-03 22:45:24,051 INFO SenderThread:1987335 [sender.py:_start_run_threads():1188] run started: 52k33ku2 with start time 1735962323.763129 +2025-01-03 22:45:24,066 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:45:24,066 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:45:24,140 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:45:24,145 DEBUG HandlerThread:1987335 [system_info.py:__init__():26] System info init +2025-01-03 22:45:24,145 DEBUG HandlerThread:1987335 [system_info.py:__init__():41] System info init done +2025-01-03 22:45:24,145 INFO HandlerThread:1987335 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:45:24,145 INFO SystemMonitor:1987335 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:45:24,146 INFO HandlerThread:1987335 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:45:24,146 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:45:24,146 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:45:24,147 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:45:24,148 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:45:24,149 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:45:24,158 DEBUG HandlerThread:1987335 [system_info.py:probe():152] Probing system +2025-01-03 22:45:24,160 DEBUG HandlerThread:1987335 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:45:24,164 DEBUG HandlerThread:1987335 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_info.py:probe():200] Probing system done +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:45:24.158820', 'startedAt': '2025-01-04T03:45:23.754665', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.738125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 4499.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.063, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 863.9541130065918}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:45:24,165 INFO HandlerThread:1987335 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:45:24,165 INFO HandlerThread:1987335 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_info.py:_save_code():44] Saving code +2025-01-03 22:45:24,166 DEBUG HandlerThread:1987335 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:45:24,166 DEBUG HandlerThread:1987335 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:45:24,228 DEBUG HandlerThread:1987335 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:45:24,228 INFO HandlerThread:1987335 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:45:24,231 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:45:24,231 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:45:24,232 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:45:24,232 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:45:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:45:24,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:45:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:24,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:24,473 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/60cew8al-wandb-metadata.json +2025-01-03 22:45:24,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:24,473 INFO HandlerThread:1987335 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/tb" + save: true +} +control { + relay_id: "78a19ede38ec44a88ed85a83a538ee6b" +} +_info { + stream_id: "52k33ku2" +} + +2025-01-03 22:45:24,474 DEBUG HandlerThread:1987335 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:45:24,492 INFO wandb-upload_1:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/9xy9zl4z-code/FISH/eval_robot.py +2025-01-03 22:45:24,500 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/cnnbt49m-diff.patch +2025-01-03 22:45:24,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: telemetry +2025-01-03 22:45:24,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: tbrecord +2025-01-03 22:45:24,622 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:45:24,622 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 with policy live +2025-01-03 22:45:24,850 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fwjmtloj-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt +2025-01-03 22:45:25,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH +2025-01-03 22:45:25,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code +2025-01-03 22:45:25,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:26,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:27,052 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:27,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:28,214 DEBUG SenderThread:1987335 [sender.py:send():379] send: config +2025-01-03 22:45:28,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:29,223 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:29,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:30,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:31,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:31,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:32,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:33,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:33,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:34,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:34,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:35,055 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:35,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:36,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:37,055 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:37,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:38,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:39,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:39,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:39,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:39,568 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:40,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:41,056 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:41,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:42,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:43,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:44,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:45,069 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:45,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:45,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:46,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:47,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:48,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:49,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:50,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:50,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:51,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:52,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:53,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:54,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:54,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:54,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:55,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:55,572 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:56,072 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:45:56,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:57,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:58,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:59,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:00,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:01,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:01,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:02,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:03,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:04,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:05,478 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:06,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:07,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:07,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:08,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:09,076 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:09,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:09,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:10,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:10,795 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:46:10,795 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 with policy now +2025-01-03 22:46:10,795 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:46:10,796 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:46:10,797 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:46:10,797 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media +2025-01-03 22:46:11,171 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/rxevysvd-media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,429 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/5s9bkhuu-media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:12,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:12,802 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:13,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:13,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:14,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:15,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:16,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:17,078 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:17,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:18,161 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:18,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:19,079 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:19,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:20,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:21,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:22,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:23,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:23,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:24,149 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:46:24,149 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:46:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:24,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:25,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:26,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:27,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:28,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:28,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:29,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:30,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:31,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:32,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:33,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:34,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:34,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:35,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:36,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:37,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:38,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:39,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:39,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:39,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:39,619 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:40,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:41,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:42,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:43,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:44,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:45,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:45,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:46,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:47,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:48,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:49,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:50,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:50,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:51,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:52,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:53,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:54,150 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:46:54,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:54,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:54,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:55,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:55,530 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:56,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:57,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:58,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:59,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:00,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:01,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:01,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:02,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:03,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:04,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:05,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:06,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:07,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:07,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:08,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:09,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:09,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:10,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:11,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:12,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:12,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:13,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:14,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:15,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:16,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:17,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:18,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:18,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:19,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:20,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:21,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:22,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:23,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:23,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:24,150 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:47:24,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:24,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:24,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:25,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:26,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:27,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:28,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:28,532 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:29,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:30,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:31,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:32,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:33,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:34,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:34,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:35,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:36,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:37,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:38,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:39,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:39,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:39,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:39,612 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:40,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:41,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:42,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:43,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:44,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:45,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:45,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:46,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:47,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:48,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:49,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:50,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:51,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:51,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:52,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:53,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:54,151 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:47:54,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:54,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:54,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:55,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:56,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:56,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:57,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:58,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:59,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:00,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:01,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:02,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:02,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:03,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:04,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:05,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:06,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:07,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:07,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:08,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:09,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:09,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:10,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:11,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:12,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:12,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:13,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:14,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:15,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:16,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:17,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:18,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:18,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:19,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:20,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:21,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:22,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:23,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:24,151 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:48:24,153 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:24,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:24,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:25,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:26,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:27,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:28,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:29,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:29,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:30,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:31,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:32,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:33,123 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:48:33,363 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/stcnm5gy-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:48:33,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:34,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:35,124 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:48:35,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:35,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:36,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:37,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:38,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:39,125 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:48:39,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:39,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:39,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:40,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:40,596 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:41,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:42,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:43,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:44,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:45,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:46,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:46,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:47,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:48,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:49,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:50,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:51,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:52,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:52,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:53,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:54,152 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:48:54,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:54,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:54,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:55,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:56,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:57,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:57,541 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:58,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:59,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:00,502 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:01,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:02,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:03,478 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:03,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:04,508 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:04,569 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:49:04,569 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:49:04,569 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 with policy now +2025-01-03 22:49:04,570 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:04,570 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:04,570 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:04,823 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/w0gv9v2t-media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:49:05,399 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/gtthix5c-media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:06,512 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:07,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:07,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:08,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:09,135 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:09,261 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:09,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:09,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:09,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:10,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:11,514 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:12,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:13,137 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:13,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:14,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:14,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:15,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:16,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:17,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:18,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:19,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:19,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:20,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:21,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:22,030 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:22,030 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:22,032 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:22,139 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:22,140 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:49:22,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:23,140 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:23,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:24,152 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:49:24,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:24,475 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:24,611 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:25,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:26,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:27,141 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:27,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:28,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:29,142 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:29,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:30,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:30,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:31,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:32,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:33,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:34,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:35,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:35,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:36,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:37,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:38,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:39,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:39,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:39,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:40,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:40,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:41,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:42,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:43,146 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:43,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:44,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:45,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:46,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:46,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:47,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:48,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:49,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:50,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:51,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:51,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:52,527 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:53,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:53,326 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:49:53,326 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 with policy now +2025-01-03 22:49:53,330 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:49:53,330 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:53,330 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:53,331 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:53,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:53,598 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fa955bxj-media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:49:54,153 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:49:54,430 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/cu5o6l11-media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:54,486 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:54,531 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:55,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:55,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:56,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:56,627 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:57,528 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:58,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:59,152 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:59,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:00,530 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:01,152 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:01,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:01,805 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:02,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:03,549 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:03,858 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:03,858 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:03,859 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:04,153 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:04,154 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:04,381 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/o7vv2xe6-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:04,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:05,154 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:05,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:06,538 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:07,155 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:07,432 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:07,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:08,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:09,155 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:09,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:09,475 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:09,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:10,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:11,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:12,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:12,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:13,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:14,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:15,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:16,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:17,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:18,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:18,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:19,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:20,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:21,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:22,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:23,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:24,154 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:50:24,154 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:24,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:24,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:24,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:25,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:26,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:27,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:28,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:29,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:29,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:30,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:31,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:32,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:33,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:34,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:35,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:35,218 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:35,549 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:35,726 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:50:35,726 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 with policy now +2025-01-03 22:50:35,728 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:50:35,729 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:35,729 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:35,729 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:35,973 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/ux2eefwg-media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:36,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,165 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:50:36,431 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/vvr8yt8y-media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:37,165 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:37,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:38,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:39,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:39,488 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:39,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:40,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:40,607 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:41,166 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:41,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:42,551 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:43,167 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:43,551 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:44,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:45,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:46,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:46,556 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:47,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:48,168 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:48,565 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:48,915 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:48,915 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:48,916 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:49,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:49,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:49,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:50,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:51,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:51,508 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:51,563 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:52,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:53,170 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:53,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:54,154 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:50:54,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:54,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:54,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:55,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:56,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:56,563 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:57,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:58,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:59,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:00,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:01,173 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:01,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:01,801 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:02,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:03,174 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:03,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:04,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:05,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:06,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:07,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:07,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:08,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:09,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:09,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:09,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:10,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:11,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:12,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:13,178 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:13,217 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:13,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:14,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:15,562 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:16,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:17,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:18,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:18,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:19,564 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:20,103 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:20,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 with policy now +2025-01-03 22:51:20,105 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:20,106 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:20,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:20,107 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:20,180 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:20,180 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:20,181 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:51:20,357 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/5o0r58z3-media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:20,567 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:20,598 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fu648ifn-media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:21,181 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:21,570 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:22,577 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:23,574 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:23,752 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:24,155 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:51:24,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:24,482 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:24,579 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:25,182 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:25,589 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:26,594 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:27,592 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:28,594 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:29,183 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:29,522 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:29,593 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:30,591 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:31,597 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:32,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:33,600 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:34,522 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:34,597 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:35,610 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:35,980 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:35,981 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:35,981 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:36,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:36,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:36,258 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:51:36,258 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:51:36,418 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:36,437 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/q3ju93tp-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:36,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:36,739 INFO wandb-upload_0:1987335 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq61gnfh2 +2025-01-03 22:51:36,838 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:51:36,846 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:37,098 INFO SenderThread:1987335 [sender.py:send_request_log_artifact():1525] logged artifact run-52k33ku2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5NzUwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MTcyMQ==', 'latestArtifact': None}} +2025-01-03 22:51:37,098 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:37,098 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,099 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send():379] send: telemetry +2025-01-03 22:51:37,100 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,100 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,100 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,100 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,101 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,102 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,102 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,103 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,103 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,104 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,104 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,104 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,104 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,105 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,105 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,105 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,105 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,106 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,106 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:51:37,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:37,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table +2025-01-03 22:51:37,287 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/4nd68nxt-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,416 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/hg3nn0ra-media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:37,495 INFO wandb-upload_1:1987335 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmph62jfnpw +2025-01-03 22:51:37,527 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/osfv52tg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:37,861 INFO SenderThread:1987335 [sender.py:send_request_log_artifact():1525] logged artifact run-52k33ku2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5NzUyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MTcyMg==', 'latestArtifact': None}} +2025-01-03 22:51:37,861 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:37,861 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json with policy now +2025-01-03 22:51:37,862 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:37,862 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,862 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:38,108 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/y5r6l74i-media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:38,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:38,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:39,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:39,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:39,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:39,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:39,627 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:40,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:41,600 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:42,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: exit +2025-01-03 22:51:42,621 INFO SenderThread:1987335 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_exit():588] handling runtime: 378 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_exit():594] send defer +2025-01-03 22:51:42,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,622 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:51:42,622 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:51:42,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:51:42,623 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,623 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:51:42,623 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:51:42,623 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:51:42,623 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:51:42,623 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:51:42,624 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:51:42,624 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:51:42,676 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:51:42,677 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:51:42,677 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,677 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:51:42,677 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:51:42,677 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,677 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:51:43,189 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:45,190 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,988 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:51:48,988 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:48,989 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:48,989 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:51:48,989 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:48,989 DEBUG SenderThread:1987335 [sender.py:send():379] send: summary +2025-01-03 22:51:48,993 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:48,993 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:51:48,994 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:48,994 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:51:48,994 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:51:48,997 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:49,171 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:51:49,171 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:49,171 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:51:49,171 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:49,171 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:51:49,191 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:51:49,191 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:49,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:51,189 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:51:51,189 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:51,189 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:51,190 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:51:51,190 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:51,190 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:51:51,190 INFO SenderThread:1987335 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:51:51,192 INFO SenderThread:1987335 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:51:51,195 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:51,195 INFO SenderThread:1987335 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:51:51,205 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:51:51,205 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:51,205 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:51:51,206 DEBUG SenderThread:1987335 [sender.py:send():379] send: artifact +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'versionIndex': 413}}} +2025-01-03 22:51:51,435 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:51:51,623 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:52,195 INFO SenderThread:1987335 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml config.yaml +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json wandb-summary.json +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt requirements.txt +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log output.log +2025-01-03 22:51:52,199 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch diff.patch +2025-01-03 22:51:52,201 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:52,204 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:52,204 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:51:52,204 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:52,204 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:52,204 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:51:52,206 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:52,206 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:51:52,206 INFO SenderThread:1987335 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:51:52,378 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt +2025-01-03 22:51:52,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:52,624 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,325 INFO wandb-upload_1:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:53,326 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:51:53,368 INFO wandb-upload_3:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/kfym2c35-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:53,375 INFO wandb-upload_4:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:53,575 INFO Thread-11 (_thread_body):1987335 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:51:53,575 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,575 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:51:53,576 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,576 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:51:53,576 INFO SenderThread:1987335 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:51:53,577 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:51:53,577 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,577 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:51:53,577 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,577 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:51:53,578 INFO SenderThread:1987335 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:51:53,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,645 INFO SenderThread:1987335 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:51:53,646 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,646 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,646 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:51:53,646 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:51:53,646 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,646 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send():379] send: final +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send():379] send: footer +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,647 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,648 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:51:53,648 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,648 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:51:53,649 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:51:53,649 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:53,713 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,713 INFO MainThread:1987335 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:51:53,713 INFO MainThread:1987335 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:51:53,714 INFO MainThread:1987335 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:51:53,714 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:51:53,714 INFO HandlerThread:1987335 [handler.py:finish():882] shutting down handler +2025-01-03 22:51:54,648 INFO WriterThread:1987335 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [sender.py:finish():1615] shutting down sender +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [file_pusher.py:join():175] waiting for file pusher diff --git a/52k33ku2/wandb/debug.log b/52k33ku2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7a159ba0fbe54e93c4224d8bbd40688fed82c4b5 --- /dev/null +++ b/52k33ku2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Configure stats pid to 1987170 +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/settings +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug.log +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug-internal.log +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2'} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():619] starting backend +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():623] setting up manager +2025-01-03 22:45:23,761 INFO MainThread:1987170 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:45:23,762 INFO MainThread:1987170 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:45:23,774 INFO MainThread:1987170 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:45:23,783 INFO MainThread:1987170 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:45:24,066 INFO MainThread:1987170 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:45:24,135 INFO MainThread:1987170 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:45:24,135 INFO MainThread:1987170 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:45:24,471 INFO MainThread:1987170 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:45:24,471 INFO MainThread:1987170 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:45:24,472 INFO MainThread:1987170 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:45:24,472 INFO MainThread:1987170 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:45:24,473 INFO MainThread:1987170 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:45:24,473 INFO MainThread:1987170 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/tb, True +2025-01-03 22:45:28,213 INFO MainThread:1987170 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:51:54,715 WARNING MsgRouterThr:1987170 [router.py:message_loop():77] message_loop has been closed diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e13edeebb159897961be9f7ababfce4c956fd6e8 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735962323 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7979e01d0a288182d9e97dcbeccffe5d96c779bc --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[28, 30, 0.9333333333333333, 0.7594559788185007, 0.99335967366961, 0.8066636919361954, 0.9915648604691375, 0.8304558693701727, 0.9808433650962447, 0.8389251864768719, 0.9808790766603611]]} \ No newline at end of file diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3ef65e5ffba24232635d0795e24d17bf755eab74 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8807ddb3aac3464696fe7106a8211d9caded6db0cac253b390c8fcc9fdc634d +size 1177467 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52eca207534b226ed7fa935c2a9b685fb7c19426 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93cac9f740e1662671287f68c5f597c67c4a08fb7e4596ce529b3b96a384d87d +size 1188790 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31cb23910e01582f8f1d042ee3ce129b4ecf98c6 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0f4e50e7a3af14dc5d1803d57a0a191579edbd167d55c1b6e7fd9482ff42bb +size 1182853 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ede54b0bee14ae59d494a4dd89e97235b2e019a --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c01cf859d0173b75d00edc1a7130eb2200d07ee87d70b56f312948359535ac32 +size 1182945 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7739eb27093838a8ffa96e2c31e7502966ec1195 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa04d437225714191e3a167bc5aed343b7e8db97f3a29339b2969c15d457517 +size 1182202 diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..493f64c80d888f12cbb122b2f4da9c90ed683201 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735962328.214428]: resetting environment +[INFO] [1735962328.219090]: cleared current plan +[INFO] [1735962328.220277]: moving to home +[INFO] [1735962329.222643]: reached home +[INFO] [1735962329.224015]: reset action history +[INFO] [1735962331.930783]: environment reset +Starting episode 0 +[INFO] [1735962331.932758]: resetting environment +[INFO] [1735962331.935432]: cleared current plan +[INFO] [1735962331.936403]: moving to home +[INFO] [1735962332.941447]: reached home +[INFO] [1735962332.942907]: reset action history +[INFO] [1735962335.652063]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735962339.061839]: resetting environment +[INFO] [1735962339.067952]: cleared current plan +[INFO] [1735962339.069623]: moving to home +[INFO] [1735962340.071893]: reached home +[INFO] [1735962340.073202]: reset action history +[INFO] [1735962342.780858]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/episode_rosbags/episode_0_2025-01-03-22-45-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1987170.log +started roslaunch server http://158.130.50.37:34743/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1987446] +process[print_text-2]: started with pid [1987470] +process[rosbag_record-3]: started with pid [1987471] +started bagging! +For topic gripper_width: timestamp difference is 100415088 for nearest: 1735962367628189660 - target: 1735962367527774572 at idx 249 +[INFO] [1735962369.800146]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735962370.528169]: Stored episode 1. +[INFO] [1735962370.528376]: Saving video... +[INFO] [1735962370.783626]: Video saved! +Starting episode 1 +[INFO] [1735962370.795945]: resetting environment +[INFO] [1735962370.800733]: cleared current plan +[INFO] [1735962370.801584]: moving to home +[INFO] [1735962374.404532]: reached home +[INFO] [1735962374.405427]: reset action history +[INFO] [1735962377.117222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962512.825976]: resetting environment +[INFO] [1735962512.829983]: cleared current plan +[INFO] [1735962512.830156]: moving to home +[INFO] [1735962513.832461]: reached home +[INFO] [1735962513.832874]: reset action history +[INFO] [1735962516.538309]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/episode_rosbags/episode_1_2025-01-03-22-48-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1987170.log +started roslaunch server http://158.130.50.37:42269/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1988201] +process[print_text-5]: started with pid [1988202] +process[rosbag_record-6]: started with pid [1988203] +started bagging! +[INFO] [1735962543.484923]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735962544.254764]: Stored episode 2. +[INFO] [1735962544.255430]: Saving video... +[INFO] [1735962544.522927]: Video saved! +Starting episode 2 +[INFO] [1735962544.565981]: resetting environment +[INFO] [1735962544.587416]: cleared current plan +[INFO] [1735962544.588153]: moving to home +[INFO] [1735962548.212369]: reached home +[INFO] [1735962548.217819]: reset action history +[INFO] [1735962550.998827]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962561.221191]: resetting environment +[INFO] [1735962561.233554]: cleared current plan +[INFO] [1735962561.234558]: moving to home +[INFO] [1735962562.238769]: reached home +[INFO] [1735962562.239484]: reset action history +[INFO] [1735962564.946669]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/episode_rosbags/episode_2_2025-01-03-22-49-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1987170.log +started roslaunch server http://158.130.50.37:38313/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1988368] +process[print_text-8]: started with pid [1988369] +process[rosbag_record-9]: started with pid [1988393] +started bagging! +For topic gripper_width: timestamp difference is -104397702 for nearest: 1735962567961521220 - target: 1735962568065918922 at idx 38 +For topic gripper_width: timestamp difference is 106884717 for nearest: 1735962581961554050 - target: 1735962581854669333 at idx 228 +[INFO] [1735962591.924155]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735962592.696478]: Stored episode 3. +[INFO] [1735962592.702432]: Saving video... +[INFO] [1735962593.286335]: Video saved! +Starting episode 3 +[INFO] [1735962593.339792]: resetting environment +[INFO] [1735962593.367539]: cleared current plan +[INFO] [1735962593.370305]: moving to home +[WARN] [1735962593.066639]: Plan exhausted +[WARN] [1735962593.105462]: Plan exhausted +[WARN] [1735962593.146963]: Plan exhausted +[WARN] [1735962593.189814]: Plan exhausted +[WARN] [1735962593.228429]: Plan exhausted +[WARN] [1735962593.268391]: Plan exhausted +[WARN] [1735962593.305703]: Plan exhausted +[WARN] [1735962593.345960]: Plan exhausted +[INFO] [1735962596.977973]: reached home +[INFO] [1735962596.983247]: reset action history +[INFO] [1735962599.764134]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962603.534584]: resetting environment +[INFO] [1735962603.553956]: cleared current plan +[INFO] [1735962603.555981]: moving to home +[INFO] [1735962604.560101]: reached home +[INFO] [1735962604.561596]: reset action history +For topic color: timestamp difference is -166534901 for nearest: 1735962607200901031 - target: 1735962607367435932 at idx 29 +[INFO] [1735962607.446714]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/episode_rosbags/episode_3_2025-01-03-22-50-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1987170.log +started roslaunch server http://158.130.50.37:43173/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1988509] +process[print_text-11]: started with pid [1988510] +process[rosbag_record-12]: started with pid [1988534] +started bagging! +[INFO] [1735962634.415265]: Storing episode... +[print_text-11] killing on exit[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735962635.217489]: Stored episode 4. +[INFO] [1735962635.224935]: Saving video... +[INFO] [1735962635.672984]: Video saved! +Starting episode 4 +[INFO] [1735962635.734540]: resetting environment +[INFO] [1735962635.769440]: cleared current plan +[INFO] [1735962635.770605]: moving to home +[WARN] [1735962635.752582]: Plan exhausted +[INFO] [1735962639.379102]: reached home +[INFO] [1735962639.398123]: reset action history +[INFO] [1735962642.252200]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962648.114337]: resetting environment +[INFO] [1735962648.146999]: cleared current plan +[INFO] [1735962648.154244]: moving to home +[INFO] [1735962649.159612]: reached home +[INFO] [1735962649.165886]: reset action history +[INFO] [1735962651.952748]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/episode_rosbags/episode_4_2025-01-03-22-50-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1987170.log +started roslaunch server http://158.130.50.37:39937/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1988672] +process[print_text-14]: started with pid [1988673] +process[rosbag_record-15]: started with pid [1988697] +started bagging! +For topic gripper_width: timestamp difference is 103353946 for nearest: 1735962660361522643 - target: 1735962660258168697 at idx 124 +For topic gripper_width: timestamp difference is 103698143 for nearest: 1735962660761546263 - target: 1735962660657848120 at idx 127 +For topic gripper_width: timestamp difference is 112142707 for nearest: 1735962671161519910 - target: 1735962671049377203 at idx 249 +[INFO] [1735962678.932337]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735962679.728540]: Stored episode 5. +[INFO] [1735962679.732165]: Saving video... +[INFO] [1735962680.043590]: Video saved! +[INFO] [1735962680.106759]: resetting environment +[INFO] [1735962680.136683]: cleared current plan +[INFO] [1735962680.142107]: moving to home +[INFO] [1735962683.751526]: reached home +[INFO] [1735962683.756894]: reset action history +[INFO] [1735962686.630367]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0d2fb0f8108428a6bc3d2d0b376a3eff6dc27da7 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:45:24.158820", + "startedAt": "2025-01-04T03:45:23.754665", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.738125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.938, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.989, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.063, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 863.9541130065918 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..860a763f92287f68090dda2d815b8929c3ff6482 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f8807ddb3aac3464696fe7106a8211d9caded6db0cac253b390c8fcc9fdc634d", "size": 1177467, "path": "media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4"}, "global_step": 663, "_timestamp": 1735962696.736319, "_runtime": 372.9731900691986, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "93cac9f740e1662671287f68c5f597c67c4a08fb7e4596ce529b3b96a384d87d", "size": 1188790, "path": "media/videos/eval/1_eval_1_93cac9f740e166267128.mp4"}, "eval/num_success": 5.0, "episode": 30.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "df0f4e50e7a3af14dc5d1803d57a0a191579edbd167d55c1b6e7fd9482ff42bb", "size": 1182853, "path": "media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c01cf859d0173b75d00edc1a7130eb2200d07ee87d70b56f312948359535ac32", "size": 1182945, "path": "media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "2aa04d437225714191e3a167bc5aed343b7e8db97f3a29339b2969c15d457517", "size": 1182202, "path": "media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://q7kb9mm6vkq2097q7rhos9seo5cz1vexj5tityro1jg4ngffxnslk0hpohtaoekqwskfdso8yeixgt60598hgko790roqztfkipqyfbk5slmef95bpjbjdvgbk29qy88/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://d0pdnn9zkxfhc3fp20qshjbi56pawn33h2g2x0xvritofrpsugpwu0bccvway5bhz69tsw6bkistrbzwix8n1mlzfrtpcil9u4wy8jbqfflp0bknvpf0fq0i39kty3sm:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 28, "total_num_episodes": 30, "total_success_rate": 0.9333333333333333, "total_success_rate_lb_0.01": 0.7594559788185007, "total_success_rate_ub_0.01": 0.99335967366961, "total_success_rate_lb_0.025": 0.8066636919361954, "total_success_rate_ub_0.025": 0.9915648604691375, "total_success_rate_lb_0.05": 0.8304558693701727, "total_success_rate_ub_0.05": 0.9808433650962447, "total_success_rate_lb_0.1": 0.8389251864768719, "total_success_rate_ub_0.1": 0.9808790766603611, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "30dbc0829a578769143833f8a2644021662bce77ed7f6e8e651ef1d1c2f94542", "size": 518, "artifact_path": "wandb-client-artifact://qe8arzzjzemj3nsecsq859hzx4i3f25jjy4ycqvimk4344hnha2dpw2mjo0342di6z7nbvslfqiwsyhry5b0swdz2hnmhh15rlnzuicqljhwlone4axpbe9w836nzffc/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://lxohiuje290kt8ahwzmhxdrztyhondl472jeux89kof054s511f91nn92p30j945s8v093uo5watzqxgicy0ptvld6mi3lfo5blmsxupsc6vjk9zs08kuhb31ihkzm35:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 378}, "eval/total_success_rate": 0.9333333373069763, "eval/total_success_rate_lb_0.01": 0.7594559788703918, "eval/total_success_rate_ub_0.01": 0.9933596849441528, "eval/total_success_rate_lb_0.025": 0.8066636919975281, "eval/total_success_rate_ub_0.025": 0.9915648698806763, "eval/total_success_rate_lb_0.05": 0.8304558396339417, "eval/total_success_rate_ub_0.05": 0.9808433651924133, "eval/total_success_rate_lb_0.1": 0.8389251828193665, "eval/total_success_rate_ub_0.1": 0.9808790683746338} \ No newline at end of file diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug-internal.log b/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8b1f90d2f40b82cbf97a1c25dcc97d5a5a942ed4 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug-internal.log @@ -0,0 +1,921 @@ +2025-01-03 22:45:23,762 INFO StreamThr :1987335 [internal.py:wandb_internal():85] W&B internal server running at pid: 1987335, started at: 2025-01-03 22:45:23.762311 +2025-01-03 22:45:23,763 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:45:23,774 INFO WriterThread:1987335 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb +2025-01-03 22:45:23,775 DEBUG SenderThread:1987335 [sender.py:send():379] send: header +2025-01-03 22:45:23,785 DEBUG SenderThread:1987335 [sender.py:send():379] send: run +2025-01-03 22:45:24,051 INFO SenderThread:1987335 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files +2025-01-03 22:45:24,051 INFO SenderThread:1987335 [sender.py:_start_run_threads():1188] run started: 52k33ku2 with start time 1735962323.763129 +2025-01-03 22:45:24,066 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:45:24,066 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:45:24,140 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:45:24,145 DEBUG HandlerThread:1987335 [system_info.py:__init__():26] System info init +2025-01-03 22:45:24,145 DEBUG HandlerThread:1987335 [system_info.py:__init__():41] System info init done +2025-01-03 22:45:24,145 INFO HandlerThread:1987335 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:45:24,145 INFO SystemMonitor:1987335 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:45:24,146 INFO HandlerThread:1987335 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:45:24,146 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:45:24,146 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:45:24,147 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:45:24,148 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:45:24,149 INFO SystemMonitor:1987335 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:45:24,158 DEBUG HandlerThread:1987335 [system_info.py:probe():152] Probing system +2025-01-03 22:45:24,160 DEBUG HandlerThread:1987335 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:45:24,164 DEBUG HandlerThread:1987335 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_info.py:probe():200] Probing system done +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:45:24.158820', 'startedAt': '2025-01-04T03:45:23.754665', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.738125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.938, 'min': 800.0, 'max': 4700.0}, {'current': 4499.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.063, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 863.9541130065918}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:45:24,165 INFO HandlerThread:1987335 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:45:24,165 INFO HandlerThread:1987335 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:45:24,165 DEBUG HandlerThread:1987335 [system_info.py:_save_code():44] Saving code +2025-01-03 22:45:24,166 DEBUG HandlerThread:1987335 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:45:24,166 DEBUG HandlerThread:1987335 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:45:24,228 DEBUG HandlerThread:1987335 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:45:24,228 INFO HandlerThread:1987335 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:45:24,231 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:45:24,231 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:45:24,232 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:45:24,232 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:45:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:45:24,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:45:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:24,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:24,473 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/60cew8al-wandb-metadata.json +2025-01-03 22:45:24,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:24,473 INFO HandlerThread:1987335 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/tb" + save: true +} +control { + relay_id: "78a19ede38ec44a88ed85a83a538ee6b" +} +_info { + stream_id: "52k33ku2" +} + +2025-01-03 22:45:24,474 DEBUG HandlerThread:1987335 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:45:24,492 INFO wandb-upload_1:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/9xy9zl4z-code/FISH/eval_robot.py +2025-01-03 22:45:24,500 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/cnnbt49m-diff.patch +2025-01-03 22:45:24,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: telemetry +2025-01-03 22:45:24,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: tbrecord +2025-01-03 22:45:24,622 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:45:24,622 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 with policy live +2025-01-03 22:45:24,850 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fwjmtloj-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json +2025-01-03 22:45:25,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt +2025-01-03 22:45:25,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH +2025-01-03 22:45:25,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code +2025-01-03 22:45:25,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:26,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:27,052 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:27,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:28,214 DEBUG SenderThread:1987335 [sender.py:send():379] send: config +2025-01-03 22:45:28,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:29,223 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:29,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:30,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:31,053 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:31,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:32,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:33,054 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:33,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:34,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:34,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:35,055 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:35,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:36,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:37,055 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:37,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:38,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:39,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:39,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:39,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:39,568 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:40,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:41,056 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:41,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:42,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:43,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:44,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:45,069 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:45:45,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:45,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:46,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:47,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:48,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:49,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:50,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:50,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:51,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:52,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:53,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:54,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:45:54,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:45:54,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:55,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:55,572 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:45:56,072 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:45:56,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:57,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:58,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:45:59,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:00,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:01,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:01,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:02,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:03,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:04,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:05,478 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:06,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:07,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:07,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:08,480 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:09,076 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:09,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:09,471 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:10,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:10,795 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:46:10,795 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 with policy now +2025-01-03 22:46:10,795 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:46:10,796 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:46:10,797 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:46:10,797 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:46:11,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media +2025-01-03 22:46:11,171 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/rxevysvd-media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,429 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/5s9bkhuu-media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:46:11,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:12,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:12,802 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:13,077 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:13,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:14,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:15,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:16,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:17,078 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:17,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:18,161 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:18,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:19,079 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:46:19,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:20,481 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:21,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:22,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:23,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:23,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:24,149 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:46:24,149 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:46:24,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:24,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:25,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:26,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:27,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:28,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:28,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:29,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:30,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:31,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:32,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:33,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:34,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:34,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:35,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:36,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:37,483 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:38,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:39,471 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:39,472 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:39,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:39,619 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:40,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:41,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:42,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:43,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:44,484 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:45,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:45,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:46,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:47,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:48,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:49,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:50,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:50,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:51,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:52,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:53,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:54,150 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:46:54,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:46:54,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:46:54,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:55,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:55,530 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:46:56,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:57,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:58,486 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:46:59,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:00,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:01,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:01,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:02,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:03,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:04,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:05,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:06,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:07,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:07,487 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:08,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:09,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:09,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:10,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:11,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:12,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:12,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:13,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:14,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:15,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:16,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:17,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:18,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:18,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:19,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:20,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:21,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:22,489 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:23,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:23,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:24,150 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:47:24,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:24,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:24,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:25,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:26,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:27,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:28,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:28,532 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:29,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:30,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:31,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:32,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:33,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:34,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:34,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:35,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:36,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:37,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:38,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:39,472 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:39,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:39,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:39,612 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:40,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:41,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:42,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:43,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:44,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:45,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:45,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:46,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:47,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:48,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:49,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:50,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:51,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:51,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:52,493 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:53,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:54,151 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:47:54,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:47:54,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:47:54,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:55,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:56,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:56,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:47:57,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:58,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:47:59,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:00,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:01,494 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:02,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:02,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:03,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:04,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:05,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:06,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:07,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:07,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:08,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:09,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:09,473 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:09,513 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:10,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:11,495 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:12,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:12,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:13,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:14,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:15,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:16,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:17,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:18,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:18,496 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:19,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:20,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:21,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:22,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:23,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:24,151 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:48:24,153 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:24,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:24,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:25,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:26,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:27,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:28,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:29,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:29,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:30,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:31,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:32,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:33,123 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:48:33,363 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/stcnm5gy-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:48:33,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:34,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:35,124 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:48:35,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:35,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:36,498 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:37,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:38,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:39,125 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:48:39,473 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:39,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:39,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:40,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:40,596 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:41,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:42,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:43,499 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:44,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:45,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:46,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:46,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:47,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:48,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:49,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:50,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:51,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:52,477 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:52,500 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:53,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:54,152 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:48:54,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:48:54,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:48:54,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:55,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:56,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:57,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:57,541 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:48:58,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:48:59,501 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:00,502 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:01,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:02,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:03,478 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:03,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:04,508 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:04,569 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:49:04,569 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:49:04,569 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 with policy now +2025-01-03 22:49:04,570 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:04,570 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:04,570 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:04,823 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/w0gv9v2t-media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:49:05,399 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/gtthix5c-media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:49:05,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:06,512 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:07,134 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:07,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:08,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:09,135 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:09,261 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:09,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:09,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:09,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:10,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:11,514 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:12,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:13,137 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:13,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:14,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:14,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:15,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:16,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:17,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:18,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:19,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:19,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:20,515 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:21,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:22,030 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:22,030 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:22,032 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:22,139 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:22,140 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:49:22,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:23,140 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:23,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:24,152 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:49:24,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:24,475 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:24,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:24,611 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:25,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:26,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:27,141 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:27,516 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:28,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:29,142 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:29,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:30,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:30,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:31,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:32,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:33,517 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:34,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:35,490 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:35,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:36,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:37,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:38,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:39,474 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:39,474 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:39,521 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:40,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:40,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:41,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:42,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:43,146 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:43,518 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:44,519 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:45,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:46,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:46,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:47,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:48,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:49,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:50,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:51,491 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:51,525 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:52,527 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:53,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:53,326 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:49:53,326 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 with policy now +2025-01-03 22:49:53,330 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:49:53,330 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:49:53,330 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:49:53,331 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:49:53,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:53,598 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fa955bxj-media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:49:54,153 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:49:54,430 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/cu5o6l11-media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:49:54,485 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:49:54,486 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:49:54,531 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:55,150 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:55,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:56,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:56,627 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:49:57,528 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:58,529 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:49:59,152 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:49:59,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:00,530 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:01,152 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:01,533 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:01,805 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:02,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:03,549 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:03,858 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:03,858 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:03,859 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:04,153 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:04,154 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:04,381 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/o7vv2xe6-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:04,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:05,154 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:05,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:06,538 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:07,155 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:07,432 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:07,534 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:08,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:09,155 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:09,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:09,475 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:09,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:10,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:11,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:12,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:12,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:13,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:14,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:15,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:16,535 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:17,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:18,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:18,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:19,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:20,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:21,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:22,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:23,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:24,154 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:50:24,154 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:24,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:24,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:24,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:25,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:26,536 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:27,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:28,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:29,492 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:29,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:30,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:31,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:32,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:33,537 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:34,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:35,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:35,218 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:35,549 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:35,726 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:50:35,726 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 with policy now +2025-01-03 22:50:35,728 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:50:35,729 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:35,729 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:35,729 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:35,973 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/ux2eefwg-media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:36,164 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,165 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:50:36,431 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/vvr8yt8y-media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:50:36,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:37,165 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:37,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:38,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:39,488 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:39,488 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:39,550 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:40,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:40,607 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:41,166 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:41,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:42,551 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:43,167 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:43,551 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:44,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:45,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:46,497 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:46,556 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:47,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:48,168 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:50:48,565 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:48,915 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:50:48,915 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:50:48,916 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:50:49,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:49,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:50:49,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:50,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:51,169 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:51,508 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:51,563 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:52,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:53,170 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:50:53,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:54,154 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:50:54,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:50:54,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:50:54,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:55,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:56,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:56,563 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:50:57,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:58,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:50:59,557 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:00,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:01,173 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:01,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:01,801 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:02,558 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:03,174 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:03,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:04,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:05,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:06,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:07,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:07,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:08,559 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:09,475 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:09,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:09,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:10,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:11,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:12,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:13,178 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:13,217 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:13,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:14,560 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:15,562 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:16,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:17,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:18,503 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:18,561 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:19,564 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:20,103 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:20,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 with policy now +2025-01-03 22:51:20,105 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:20,106 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:20,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:20,107 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:20,180 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:20,180 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:20,181 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval +2025-01-03 22:51:20,357 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/5o0r58z3-media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:20,567 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:20,598 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/fu648ifn-media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:21,181 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:21,570 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:22,577 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:23,574 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:23,752 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:24,155 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:51:24,482 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:24,482 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:24,579 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:25,182 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:25,589 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:26,594 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:27,592 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:28,594 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:29,183 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:29,522 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:29,593 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:30,591 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:31,597 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:32,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:33,600 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:34,522 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:34,597 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:35,610 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:35,980 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:35,981 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:35,981 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:36,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:36,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:36,258 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:51:36,258 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:51:36,418 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:36,437 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/q3ju93tp-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:36,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:36,739 INFO wandb-upload_0:1987335 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq61gnfh2 +2025-01-03 22:51:36,838 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:51:36,846 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:51:37,098 INFO SenderThread:1987335 [sender.py:send_request_log_artifact():1525] logged artifact run-52k33ku2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5NzUwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MTcyMQ==', 'latestArtifact': None}} +2025-01-03 22:51:37,098 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:37,098 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,099 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,099 DEBUG SenderThread:1987335 [sender.py:send():379] send: telemetry +2025-01-03 22:51:37,100 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,100 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,100 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,100 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,101 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,102 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,102 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,103 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,103 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,103 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,104 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,104 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,104 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,104 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,105 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,105 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,105 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,105 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,106 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,106 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:37,106 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:51:37,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:37,186 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval +2025-01-03 22:51:37,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table +2025-01-03 22:51:37,287 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/4nd68nxt-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,416 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/hg3nn0ra-media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:37,495 INFO wandb-upload_1:1987335 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmph62jfnpw +2025-01-03 22:51:37,527 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/osfv52tg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:37,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:37,861 INFO SenderThread:1987335 [sender.py:send_request_log_artifact():1525] logged artifact run-52k33ku2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5NzUyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MTcyMg==', 'latestArtifact': None}} +2025-01-03 22:51:37,861 DEBUG SenderThread:1987335 [sender.py:send():379] send: files +2025-01-03 22:51:37,861 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json with policy now +2025-01-03 22:51:37,862 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:37,862 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:37,862 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:38,108 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/y5r6l74i-media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:38,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:38,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:39,187 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:39,476 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:51:39,476 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:51:39,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:39,627 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:40,599 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:41,600 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:42,621 DEBUG SenderThread:1987335 [sender.py:send():379] send: exit +2025-01-03 22:51:42,621 INFO SenderThread:1987335 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_exit():588] handling runtime: 378 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_exit():594] send defer +2025-01-03 22:51:42,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,622 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:51:42,622 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:51:42,622 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:51:42,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:51:42,623 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,623 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:51:42,623 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:51:42,623 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:51:42,623 INFO HandlerThread:1987335 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:51:42,623 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:51:42,623 DEBUG SystemMonitor:1987335 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:51:42,624 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:51:42,624 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:51:42,676 INFO HandlerThread:1987335 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:51:42,676 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:51:42,677 DEBUG SenderThread:1987335 [sender.py:send():379] send: stats +2025-01-03 22:51:42,677 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,677 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:51:42,677 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:51:42,677 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:51:42,677 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:42,677 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:51:43,189 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:45,190 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:48,987 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,988 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:51:48,988 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send():379] send: history +2025-01-03 22:51:48,988 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:51:48,989 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:48,989 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:51:48,989 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:48,989 DEBUG SenderThread:1987335 [sender.py:send():379] send: summary +2025-01-03 22:51:48,993 INFO SenderThread:1987335 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:51:48,993 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:51:48,994 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:48,994 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:51:48,994 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:48,994 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:51:48,997 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:51:49,171 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:51:49,171 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:49,171 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:51:49,171 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:49,171 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:51:49,191 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:51:49,191 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:49,622 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:51,189 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:51:51,189 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:51,189 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:51,190 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:51:51,190 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:51,190 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:51:51,190 INFO SenderThread:1987335 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:51:51,192 INFO SenderThread:1987335 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:51:51,195 INFO Thread-12 :1987335 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:51,195 INFO SenderThread:1987335 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:51:51,205 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:51:51,205 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:51,205 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:51:51,206 DEBUG SenderThread:1987335 [sender.py:send():379] send: artifact +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'versionIndex': 413}}} +2025-01-03 22:51:51,435 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:51:51,435 INFO SenderThread:1987335 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:51:51,623 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:52,195 INFO SenderThread:1987335 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml config.yaml +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json wandb-summary.json +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt requirements.txt +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:52,196 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log output.log +2025-01-03 22:51:52,199 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/diff.patch diff.patch +2025-01-03 22:51:52,201 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 media/videos/eval/2_eval_2_df0f4e50e7a3af14dc5d.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 media/videos/eval/0_eval_0_f8807ddb3aac3464696f.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 media/videos/eval/4_eval_4_2aa04d437225714191e3.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 media/videos/eval/1_eval_1_93cac9f740e166267128.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 media/videos/eval/3_eval_3_c01cf859d0173b75d00e.mp4 +2025-01-03 22:51:52,202 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json media/table/eval/total_success_rate_ci_6_30dbc0829a5787691438.table.json +2025-01-03 22:51:52,204 INFO SenderThread:1987335 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:51:52,204 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:51:52,204 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:52,204 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:52,204 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:51:52,206 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:52,206 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:51:52,206 INFO SenderThread:1987335 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:51:52,378 INFO wandb-upload_0:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/requirements.txt +2025-01-03 22:51:52,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:52,624 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,325 INFO wandb-upload_1:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/wandb-summary.json +2025-01-03 22:51:53,326 INFO wandb-upload_2:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/config.yaml +2025-01-03 22:51:53,368 INFO wandb-upload_3:1987335 [upload_job.py:push():130] Uploaded file /tmp/tmpb9z9_5tzwandb/kfym2c35-events.out.tfevents.1735962324.leonmkim-ROG-Strix-G15CS-G15CS.1987170.0 +2025-01-03 22:51:53,375 INFO wandb-upload_4:1987335 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/files/output.log +2025-01-03 22:51:53,575 INFO Thread-11 (_thread_body):1987335 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:51:53,575 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,575 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:51:53,576 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,576 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:51:53,576 INFO SenderThread:1987335 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:51:53,577 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:51:53,577 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,577 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:51:53,577 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,577 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:51:53,578 INFO SenderThread:1987335 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:51:53,624 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,645 INFO SenderThread:1987335 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:51:53,646 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,646 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,646 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:51:53,646 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:51:53,646 INFO SenderThread:1987335 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:51:53,646 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:51:53,646 INFO HandlerThread:1987335 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send():379] send: final +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send():379] send: footer +2025-01-03 22:51:53,647 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: defer +2025-01-03 22:51:53,647 INFO SenderThread:1987335 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:51:53,647 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:51:53,648 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:51:53,648 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,648 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:51:53,649 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:51:53,649 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:51:53,713 DEBUG SenderThread:1987335 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:51:53,713 INFO MainThread:1987335 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:51:53,713 INFO MainThread:1987335 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:51:53,714 INFO MainThread:1987335 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:51:53,714 DEBUG HandlerThread:1987335 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:51:53,714 INFO HandlerThread:1987335 [handler.py:finish():882] shutting down handler +2025-01-03 22:51:54,648 INFO WriterThread:1987335 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [sender.py:finish():1615] shutting down sender +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:51:54,713 INFO SenderThread:1987335 [file_pusher.py:join():175] waiting for file pusher diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug.log b/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7a159ba0fbe54e93c4224d8bbd40688fed82c4b5 --- /dev/null +++ b/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Configure stats pid to 1987170 +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/settings +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug.log +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/wandb/run-20250103_224523-52k33ku2/logs/debug-internal.log +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2'} +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():619] starting backend +2025-01-03 22:45:23,757 INFO MainThread:1987170 [wandb_init.py:init():623] setting up manager +2025-01-03 22:45:23,761 INFO MainThread:1987170 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:45:23,762 INFO MainThread:1987170 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:45:23,774 INFO MainThread:1987170 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:45:23,783 INFO MainThread:1987170 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:45:24,066 INFO MainThread:1987170 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:45:24,135 INFO MainThread:1987170 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:45:24,135 INFO MainThread:1987170 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:45:24,471 INFO MainThread:1987170 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:45:24,471 INFO MainThread:1987170 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:45:24,472 INFO MainThread:1987170 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:45:24,472 INFO MainThread:1987170 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:45:24,473 INFO MainThread:1987170 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:45:24,473 INFO MainThread:1987170 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/52k33ku2/tb, True +2025-01-03 22:45:28,213 INFO MainThread:1987170 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:51:54,715 WARNING MsgRouterThr:1987170 [router.py:message_loop():77] message_loop has been closed diff --git a/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb b/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9011232c77b433082818001809dacfd7f946363e Binary files /dev/null and b/52k33ku2/wandb/run-20250103_224523-52k33ku2/run-52k33ku2.wandb differ diff --git a/5q1nh8a1/.hydra/config.yaml b/5q1nh8a1/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/5q1nh8a1/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/5q1nh8a1/.hydra/hydra.yaml b/5q1nh8a1/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1855903bb6bcd76867d6f13c70608f8bb7cee50 --- /dev/null +++ b/5q1nh8a1/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/5q1nh8a1/.hydra/overrides.yaml b/5q1nh8a1/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/5q1nh8a1/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/5q1nh8a1/episode_rosbags/aligned_depth_to_color_K.npy b/5q1nh8a1/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5q1nh8a1/episode_rosbags/cam_tf_world.npy b/5q1nh8a1/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/5q1nh8a1/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/5q1nh8a1/episode_rosbags/color_K.npy b/5q1nh8a1/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/5q1nh8a1/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/5q1nh8a1/episode_rosbags/depth_K.npy b/5q1nh8a1/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/5q1nh8a1/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/5q1nh8a1/eval_robot.log b/5q1nh8a1/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..95e3ab188607800a9604665dc7595836f07112c3 --- /dev/null +++ b/5q1nh8a1/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:10:36,879][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:10:36,882][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/5q1nh8a1/eval_video/0_eval.mp4 b/5q1nh8a1/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7edb3100b51c601a6d666f5a1790e1c1424b12c --- /dev/null +++ b/5q1nh8a1/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6345377befcb792e0c92371ad83ae363c65001343a244698a089ecd1cbae74c9 +size 1148631 diff --git a/5q1nh8a1/eval_video/1_eval.mp4 b/5q1nh8a1/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eec90a36b0fb86a0da240b1731f6aacf1006ad9b --- /dev/null +++ b/5q1nh8a1/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ad851b779419a67be3aa1bb6defe2d621f28df4902cee5d6863f8824d36ed7 +size 1140075 diff --git a/5q1nh8a1/eval_video/2_eval.mp4 b/5q1nh8a1/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..496a47250b129d7e8eb351d3787ab9c84892a7bf --- /dev/null +++ b/5q1nh8a1/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026bd92e4398e9ac5f9003d3135d96d78d16921a79b9bf6b022a5b6d473eeba3 +size 1136822 diff --git a/5q1nh8a1/eval_video/3_eval.mp4 b/5q1nh8a1/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cdb4e2ff43af98dd0e9c44b33fe155f134f6b54 --- /dev/null +++ b/5q1nh8a1/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcaa4b7b876ccb5c2cf6c134a59a92fe080cef90b4524c00540aae6779b8f695 +size 1136449 diff --git a/5q1nh8a1/eval_video/4_eval.mp4 b/5q1nh8a1/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7bc1cdcb4a008a949099aa25a8a9658f140b63e6 --- /dev/null +++ b/5q1nh8a1/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ceef17a9c763063daba345a605c90e4fa65883e97b051371c35eae6c3d5c66 +size 1132688 diff --git a/5q1nh8a1/tb/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 b/5q1nh8a1/tb/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 new file mode 100644 index 0000000000000000000000000000000000000000..934ce11c2c39f3c7ef7d3ef22d8102477483720f --- /dev/null +++ b/5q1nh8a1/tb/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac9cfa18f21fa33caa847859267c2ea1ed4775c5149bb55a8f2f12e0d8130e9f +size 6539 diff --git a/5q1nh8a1/wandb/debug-internal.log b/5q1nh8a1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e371b5618e9425bdbe723775b46891f446202b12 --- /dev/null +++ b/5q1nh8a1/wandb/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-03 22:10:42,890 INFO StreamThr :1979941 [internal.py:wandb_internal():85] W&B internal server running at pid: 1979941, started at: 2025-01-03 22:10:42.890609 +2025-01-03 22:10:42,892 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:10:42,903 INFO WriterThread:1979941 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb +2025-01-03 22:10:42,903 DEBUG SenderThread:1979941 [sender.py:send():379] send: header +2025-01-03 22:10:42,911 DEBUG SenderThread:1979941 [sender.py:send():379] send: run +2025-01-03 22:10:43,184 INFO SenderThread:1979941 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files +2025-01-03 22:10:43,184 INFO SenderThread:1979941 [sender.py:_start_run_threads():1188] run started: 5q1nh8a1 with start time 1735960242.89071 +2025-01-03 22:10:43,200 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:10:43,201 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:10:43,278 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:10:43,282 DEBUG HandlerThread:1979941 [system_info.py:__init__():26] System info init +2025-01-03 22:10:43,282 DEBUG HandlerThread:1979941 [system_info.py:__init__():41] System info init done +2025-01-03 22:10:43,283 INFO HandlerThread:1979941 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:10:43,283 INFO SystemMonitor:1979941 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:10:43,283 INFO HandlerThread:1979941 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:10:43,283 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:10:43,284 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:10:43,285 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:10:43,285 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:10:43,286 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:10:43,296 DEBUG HandlerThread:1979941 [system_info.py:probe():152] Probing system +2025-01-03 22:10:43,297 DEBUG HandlerThread:1979941 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:probe():200] Probing system done +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:10:43.296305', 'startedAt': '2025-01-04T03:10:42.882408', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.497125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 824.9600524902344}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:10:43,302 INFO HandlerThread:1979941 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:10:43,302 INFO HandlerThread:1979941 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:_save_code():44] Saving code +2025-01-03 22:10:43,304 DEBUG HandlerThread:1979941 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:10:43,304 DEBUG HandlerThread:1979941 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:10:43,365 DEBUG HandlerThread:1979941 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:10:43,365 INFO HandlerThread:1979941 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:10:43,368 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:10:43,368 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:10:43,369 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:10:43,369 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:10:43,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:10:43,616 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:10:43,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:43,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:43,630 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/wh68wajm-diff.patch +2025-01-03 22:10:43,640 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/r76q07uj-code/FISH/eval_robot.py +2025-01-03 22:10:43,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:43,661 INFO HandlerThread:1979941 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/tb" + save: true +} +control { + relay_id: "d6dc34db7dbd4fd89138d7a373b283a2" +} +_info { + stream_id: "5q1nh8a1" +} + +2025-01-03 22:10:43,661 DEBUG HandlerThread:1979941 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:10:43,767 DEBUG SenderThread:1979941 [sender.py:send():379] send: telemetry +2025-01-03 22:10:43,767 DEBUG SenderThread:1979941 [sender.py:send():379] send: tbrecord +2025-01-03 22:10:43,768 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:10:43,768 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 with policy live +2025-01-03 22:10:43,956 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/p69umh6m-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH +2025-01-03 22:10:44,313 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/4wcn4b1s-wandb-metadata.json +2025-01-03 22:10:44,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:45,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:46,185 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:46,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:47,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:48,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:48,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:48,993 DEBUG SenderThread:1979941 [sender.py:send():379] send: config +2025-01-03 22:10:49,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:50,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:50,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:51,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:52,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:52,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:53,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:53,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:54,188 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:54,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:55,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:56,188 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:56,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:57,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:58,189 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:58,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:58,616 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:58,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:58,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:59,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:00,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:01,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:02,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:03,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:04,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:04,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:05,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:06,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:07,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:08,192 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:08,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:09,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:09,894 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:10,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:11,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:12,193 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:12,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:13,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:13,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:13,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:14,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:15,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:15,728 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:16,194 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:11:16,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:17,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:18,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:19,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:20,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:21,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:21,769 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:22,196 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:22,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:23,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:24,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:25,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:26,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:27,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:27,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:28,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:28,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:28,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:29,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:30,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:31,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:32,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:32,769 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:33,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:34,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:35,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:36,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:37,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:37,837 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:37,847 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:11:37,847 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 with policy now +2025-01-03 22:11:37,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:11:37,849 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:11:37,849 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:11:37,849 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:11:38,110 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0wstjb2o-media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:11:38,461 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/asfi8i93-media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:39,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:40,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:40,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:41,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:42,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:43,286 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:11:43,286 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:11:43,287 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:43,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:43,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:43,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:44,202 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:44,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:45,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:46,203 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:46,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:47,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:48,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:48,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:49,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:50,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:51,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:52,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:53,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:54,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:54,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:55,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:56,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:57,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:58,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:58,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:58,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:59,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:59,722 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:00,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:01,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:02,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:03,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:04,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:05,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:05,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:06,209 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:06,455 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/hrko8ho4-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:06,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:07,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:08,210 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:08,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:09,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:10,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:10,705 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:11,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:12,211 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:12,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:13,287 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:12:13,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:13,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:13,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:14,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:15,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:15,739 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:16,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:17,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:18,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:19,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:20,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:21,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:21,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:22,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:23,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:24,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:25,633 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:26,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:26,639 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:27,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:28,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:28,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:28,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:29,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:30,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:31,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:32,217 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:32,335 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:32,651 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:33,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:34,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:35,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:36,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:37,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:37,840 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:37,891 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:12:37,891 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 with policy now +2025-01-03 22:12:37,893 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:12:37,894 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:12:37,894 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:12:37,896 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:12:38,166 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/ztwsugy4-media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:12:38,479 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zcz0lrie-media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,659 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:39,660 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:40,220 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:40,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:41,660 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:42,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:43,287 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:12:43,288 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:43,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:43,626 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:43,669 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:44,221 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:44,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:45,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:46,222 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:46,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:47,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:48,319 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:48,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:49,082 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:12:49,082 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:12:49,083 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:12:49,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:12:49,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:49,663 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:50,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:50,665 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:51,663 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:52,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:53,325 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:53,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:54,224 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:54,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:55,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:56,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:57,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:58,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:58,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:58,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:58,751 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:59,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:00,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:01,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:02,227 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:02,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:03,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:04,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:04,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:05,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:06,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:07,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:08,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:09,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:10,057 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:10,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:11,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:12,230 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:12,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:13,288 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:13:13,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:13,619 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:13,671 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:14,669 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:15,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:15,680 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:16,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:17,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:18,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:19,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:20,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:20,335 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:13:20,335 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 with policy now +2025-01-03 22:13:20,338 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:13:20,338 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:13:20,338 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:13:20,339 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:13:20,607 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/kri8wm7e-media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:20,673 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:13:21,365 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:21,540 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/pf09ehnf-media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:21,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:22,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:22,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:23,686 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:24,679 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:25,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:26,235 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:26,677 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:26,962 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:27,826 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:28,236 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:28,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:28,628 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:28,819 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:29,816 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:30,818 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:31,823 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:32,785 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:32,835 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:33,834 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:34,837 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:35,829 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:36,830 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:37,832 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:38,633 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:38,834 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:39,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:40,839 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:41,840 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:42,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:43,288 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:13:43,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:43,620 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:43,711 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:43,836 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:44,839 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:45,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:46,844 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:47,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:47,932 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:13:47,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:13:47,933 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:13:48,484 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/w1jmyvd1-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:13:48,846 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:49,637 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:49,846 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:50,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:50,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:51,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:52,243 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:52,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:53,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:54,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:55,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:55,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:56,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:57,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:58,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:58,620 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:58,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:59,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:00,246 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:00,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:01,169 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:01,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:02,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:03,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:04,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:05,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:06,177 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:06,248 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:06,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:07,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:08,249 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:08,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:09,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:10,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:11,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:11,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:12,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:13,289 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:14:13,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:13,621 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:13,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:14,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:15,854 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:16,701 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:16,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:17,853 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:18,857 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:19,320 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:14:19,320 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 with policy now +2025-01-03 22:14:19,324 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:14:19,324 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:14:19,324 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:14:19,325 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:14:19,619 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zrlmijw0-media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:19,857 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:20,253 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:14:20,558 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/clgg49tw-media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:20,856 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:21,862 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:22,361 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:22,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:23,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:24,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:24,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:25,871 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:26,866 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:27,871 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:27,921 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:28,256 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:28,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:28,628 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:28,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:29,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:30,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:31,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:31,984 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:14:31,984 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:14:31,985 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:14:32,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:33,565 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:33,880 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:34,258 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:34,874 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:35,874 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:36,258 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:36,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:37,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:38,259 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:38,635 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:38,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:39,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:40,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:41,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:42,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:43,289 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:14:43,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:43,622 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:43,752 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:43,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:44,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:45,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:46,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:47,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:48,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:49,635 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:49,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:50,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:51,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:52,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:53,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:54,636 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:54,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:55,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:56,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:57,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:58,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:58,631 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:58,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:59,723 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:59,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:00,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:01,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:02,882 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:03,705 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:03,705 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 with policy now +2025-01-03 22:15:03,711 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:03,712 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:03,712 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:03,713 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:03,879 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:03,967 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0tw0g7tt-media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:15:04,536 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0iiwb453-media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,729 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:04,883 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:05,887 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:06,269 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:06,900 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:07,897 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:08,269 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:08,887 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:09,888 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:10,248 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:10,892 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:11,897 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:12,271 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:12,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:13,290 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:15:13,641 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:15:13,641 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:15:13,901 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:14,905 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:15,797 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:15,903 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:16,896 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:17,899 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:18,273 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:18,542 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/qjabw0tk-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:18,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:15:18,674 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:15:18,815 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:18,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table +2025-01-03 22:15:19,450 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/yfrkg50a-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:19,542 INFO wandb-upload_0:1979941 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgdiy5yak +2025-01-03 22:15:19,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:15:19,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:19,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:19,924 INFO SenderThread:1979941 [sender.py:send_request_log_artifact():1525] logged artifact run-5q1nh8a1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2MDY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTE2NA==', 'latestArtifact': None}} +2025-01-03 22:15:19,924 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:19,924 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,925 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,929 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,929 DEBUG SenderThread:1979941 [sender.py:send():379] send: telemetry +2025-01-03 22:15:19,929 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,930 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,930 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,931 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,931 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,931 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,931 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,932 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,932 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,933 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,934 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,935 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,935 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,936 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,936 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,937 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,938 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:15:20,192 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/axlvm1rp-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:20,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval +2025-01-03 22:15:20,323 INFO wandb-upload_0:1979941 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_uhd4pk9 +2025-01-03 22:15:20,444 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/lmay8m1c-media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:20,737 INFO SenderThread:1979941 [sender.py:send_request_log_artifact():1525] logged artifact run-5q1nh8a1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2MDcyMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTE2Nw==', 'latestArtifact': None}} +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:20,737 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json with policy now +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:20,738 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:20,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:20,978 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zttv0tae-media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:21,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:21,738 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:21,913 DEBUG SenderThread:1979941 [sender.py:send():379] send: exit +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():588] handling runtime: 278 +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():594] send defer +2025-01-03 22:15:21,914 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,914 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:15:21,914 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:15:21,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:15:21,915 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:15:21,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:15:21,916 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:15:21,916 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:15:21,916 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:15:21,916 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,968 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:15:21,968 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:15:21,968 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,968 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,969 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:15:21,969 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:15:21,969 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,969 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:15:22,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:22,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:24,276 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:28,050 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:28,050 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:28,051 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,052 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:15:28,052 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:28,053 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,053 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:15:28,053 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:28,053 DEBUG SenderThread:1979941 [sender.py:send():379] send: summary +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:28,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:15:28,054 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,054 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:15:28,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:15:28,058 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:28,179 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:15:28,180 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,180 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:15:28,180 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,180 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:15:28,277 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:15:28,278 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:28,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:32,054 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:15:32,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:32,055 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:32,055 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:15:32,055 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:32,055 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:15:32,055 INFO SenderThread:1979941 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:15:32,057 INFO SenderThread:1979941 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:15:32,059 INFO SenderThread:1979941 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:15:32,069 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:15:32,069 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:32,069 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:15:32,069 DEBUG SenderThread:1979941 [sender.py:send():379] send: artifact +2025-01-03 22:15:32,279 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:32,377 INFO SenderThread:1979941 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'versionIndex': 411}}} +2025-01-03 22:15:32,378 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:32,378 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:15:32,378 INFO SenderThread:1979941 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:15:32,916 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml config.yaml +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json wandb-summary.json +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt requirements.txt +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:33,281 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log output.log +2025-01-03 22:15:33,281 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch diff.patch +2025-01-03 22:15:33,285 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:15:33,289 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,289 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,289 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:15:33,290 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,290 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:15:33,290 INFO SenderThread:1979941 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:15:33,461 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt +2025-01-03 22:15:33,468 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:15:33,511 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:33,525 INFO wandb-upload_4:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:33,573 INFO wandb-upload_3:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/cya0lkoz-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:33,773 INFO Thread-11 (_thread_body):1979941 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:15:33,773 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,773 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:15:33,773 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,773 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:15:33,773 INFO SenderThread:1979941 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:15:33,775 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,775 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:15:33,775 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:15:33,843 INFO SenderThread:1979941 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:15:33,843 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:15:33,843 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,843 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send():379] send: final +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send():379] send: footer +2025-01-03 22:15:33,844 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,844 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,845 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,846 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:15:33,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:15:33,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:15:33,848 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:33,898 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,898 INFO MainThread:1979941 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:15:33,898 INFO MainThread:1979941 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:15:33,899 INFO MainThread:1979941 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:15:33,899 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:15:33,899 INFO HandlerThread:1979941 [handler.py:finish():882] shutting down handler +2025-01-03 22:15:34,847 INFO WriterThread:1979941 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [sender.py:finish():1615] shutting down sender +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [file_pusher.py:join():175] waiting for file pusher diff --git a/5q1nh8a1/wandb/debug.log b/5q1nh8a1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bfb6f5a5592d45e2bb76c009a40119c59509418b --- /dev/null +++ b/5q1nh8a1/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Configure stats pid to 1979775 +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/settings +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug.log +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug-internal.log +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1'} +2025-01-03 22:10:42,885 INFO MainThread:1979775 [wandb_init.py:init():619] starting backend +2025-01-03 22:10:42,885 INFO MainThread:1979775 [wandb_init.py:init():623] setting up manager +2025-01-03 22:10:42,889 INFO MainThread:1979775 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:10:42,890 INFO MainThread:1979775 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:10:42,902 INFO MainThread:1979775 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:10:42,909 INFO MainThread:1979775 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:10:43,200 INFO MainThread:1979775 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:10:43,270 INFO MainThread:1979775 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:10:43,270 INFO MainThread:1979775 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:10:43,619 INFO MainThread:1979775 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:10:43,619 INFO MainThread:1979775 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/tb, True +2025-01-03 22:10:48,992 INFO MainThread:1979775 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:15:34,900 WARNING MsgRouterThr:1979775 [router.py:message_loop():77] message_loop has been closed diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..269e7df50fc5a9bb3c291863ad73e210bd15b96c --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735960242 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..77c852fe9f419930a0fb76a7914e3aa04a6cbbb8 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..06bbd87 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json new file mode 100644 index 0000000000000000000000000000000000000000..856c7f95f1cba01502ffabac47dd0af92bfd0f34 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[18, 20, 0.9, 0.6569254994078544, 0.9899633600327942, 0.7210156619106765, 0.9872633720704922, 0.7536698430269255, 0.9710622577917205, 0.7647808491654122, 0.9711993922468375]]} \ No newline at end of file diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7edb3100b51c601a6d666f5a1790e1c1424b12c --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6345377befcb792e0c92371ad83ae363c65001343a244698a089ecd1cbae74c9 +size 1148631 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eec90a36b0fb86a0da240b1731f6aacf1006ad9b --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ad851b779419a67be3aa1bb6defe2d621f28df4902cee5d6863f8824d36ed7 +size 1140075 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..496a47250b129d7e8eb351d3787ab9c84892a7bf --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026bd92e4398e9ac5f9003d3135d96d78d16921a79b9bf6b022a5b6d473eeba3 +size 1136822 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cdb4e2ff43af98dd0e9c44b33fe155f134f6b54 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcaa4b7b876ccb5c2cf6c134a59a92fe080cef90b4524c00540aae6779b8f695 +size 1136449 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7bc1cdcb4a008a949099aa25a8a9658f140b63e6 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ceef17a9c763063daba345a605c90e4fa65883e97b051371c35eae6c3d5c66 +size 1132688 diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..64a4b89d8642aeded15883ffb1d71f5f19c63f78 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735960248.993481]: resetting environment +[INFO] [1735960248.996702]: cleared current plan +[INFO] [1735960248.999656]: moving to home +[INFO] [1735960250.003450]: reached home +[INFO] [1735960250.006354]: reset action history +[INFO] [1735960252.715560]: environment reset +Starting episode 0 +[INFO] [1735960252.719996]: resetting environment +[INFO] [1735960252.724784]: cleared current plan +[INFO] [1735960252.726785]: moving to home +[INFO] [1735960253.729200]: reached home +[INFO] [1735960253.730907]: reset action history +[INFO] [1735960256.437906]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735960266.177259]: resetting environment +[INFO] [1735960266.182896]: cleared current plan +[INFO] [1735960266.183876]: moving to home +[INFO] [1735960267.185486]: reached home +[INFO] [1735960267.186759]: reset action history +[INFO] [1735960269.893738]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/episode_rosbags/episode_0_2025-01-03-22-11-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1979775.log +started roslaunch server http://158.130.50.37:36137/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1980051] +process[print_text-2]: started with pid [1980075] +process[rosbag_record-3]: started with pid [1980076] +started bagging! +For topic gripper_width: timestamp difference is 114937323 for nearest: 1735960280728209752 - target: 1735960280613272429 at idx 163 +[INFO] [1735960296.904072]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735960297.627070]: Stored episode 1. +[INFO] [1735960297.629678]: Saving video... +[INFO] [1735960297.836904]: Video saved! +Starting episode 1 +[INFO] [1735960297.848173]: resetting environment +[INFO] [1735960297.861020]: cleared current plan +[INFO] [1735960297.861240]: moving to home +[INFO] [1735960301.365680]: reached home +[INFO] [1735960301.366119]: reset action history +[INFO] [1735960304.074170]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960325.702496]: resetting environment +[INFO] [1735960325.704947]: cleared current plan +[INFO] [1735960325.705345]: moving to home +[INFO] [1735960326.706906]: reached home +[INFO] [1735960326.707454]: reset action history +[INFO] [1735960329.414572]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/episode_rosbags/episode_1_2025-01-03-22-12-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1979775.log +started roslaunch server http://158.130.50.37:45617/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1980187] +process[print_text-5]: started with pid [1980188] +process[rosbag_record-6]: started with pid [1980212] +started bagging! +For topic gripper_width: timestamp difference is 104274952 for nearest: 1735960350128207171 - target: 1735960350023932219 at idx 249 +For topic gripper_width: timestamp difference is 104437643 for nearest: 1735960350328209215 - target: 1735960350223771572 at idx 249 +[INFO] [1735960356.490285]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735960357.244631]: Stored episode 2. +[INFO] [1735960357.244895]: Saving video... +[WARN] [1735960357.850298]: Plan exhausted +[WARN] [1735960357.890310]: Plan exhausted +[INFO] [1735960357.839402]: Video saved! +Starting episode 2 +[INFO] [1735960357.898023]: resetting environment +[INFO] [1735960357.916693]: cleared current plan +[INFO] [1735960357.921354]: moving to home +[INFO] [1735960361.735684]: reached home +[INFO] [1735960361.737530]: reset action history +[INFO] [1735960364.513213]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960368.334595]: resetting environment +[INFO] [1735960368.349061]: cleared current plan +[INFO] [1735960368.350728]: moving to home +[INFO] [1735960369.352600]: reached home +[INFO] [1735960369.353131]: reset action history +[INFO] [1735960372.061342]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/episode_rosbags/episode_2_2025-01-03-22-12-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1979775.log +started roslaunch server http://158.130.50.37:34883/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1980325] +process[print_text-8]: started with pid [1980326] +process[rosbag_record-9]: started with pid [1980350] +started bagging! +For topic gripper_width: timestamp difference is -105334898 for nearest: 1735960379261576752 - target: 1735960379366911650 at idx 95 +For topic gripper_width: timestamp difference is -104865406 for nearest: 1735960379861540701 - target: 1735960379966406107 at idx 100 +For topic gripper_width: timestamp difference is -102723349 for nearest: 1735960389861566555 - target: 1735960389964289904 at idx 240 +[INFO] [1735960399.035000]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735960399.810197]: Stored episode 3. +[INFO] [1735960399.816532]: Saving video... +[INFO] [1735960400.294532]: Video saved! +Starting episode 3 +[INFO] [1735960400.342227]: resetting environment +[INFO] [1735960400.363060]: cleared current plan +[INFO] [1735960400.364910]: moving to home +[INFO] [1735960404.167391]: reached home +[INFO] [1735960404.173423]: reset action history +[INFO] [1735960406.961784]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960427.379613]: resetting environment +[INFO] [1735960427.391323]: cleared current plan +[INFO] [1735960427.394311]: moving to home +[INFO] [1735960428.398468]: reached home +[INFO] [1735960428.413638]: reset action history +[INFO] [1735960431.132150]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/episode_rosbags/episode_3_2025-01-03-22-13-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1979775.log +started roslaunch server http://158.130.50.37:45439/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1980479] +process[print_text-11]: started with pid [1980480] +process[rosbag_record-12]: started with pid [1980504] +started bagging! +For topic gripper_width: timestamp difference is 100790701 for nearest: 1735960439128207169 - target: 1735960439027416468 at idx 107 +For topic gripper_width: timestamp difference is 113058022 for nearest: 1735960445128206900 - target: 1735960445015148878 at idx 189 +For topic gripper_width: timestamp difference is 115916460 for nearest: 1735960446528207748 - target: 1735960446412291288 at idx 209 +[INFO] [1735960458.103367]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735960458.893755]: Stored episode 4. +[INFO] [1735960458.898280]: Saving video... +[INFO] [1735960459.260951]: Video saved! +Starting episode 4 +[INFO] [1735960459.334708]: resetting environment +[INFO] [1735960459.360603]: cleared current plan +[INFO] [1735960459.360853]: moving to home +[INFO] [1735960463.065607]: reached home +[INFO] [1735960463.085072]: reset action history +[INFO] [1735960465.872618]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960471.485405]: resetting environment +[INFO] [1735960471.511482]: cleared current plan +[INFO] [1735960471.514390]: moving to home +[INFO] [1735960472.517206]: reached home +[INFO] [1735960472.523678]: reset action history +[INFO] [1735960475.317715]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/episode_rosbags/episode_4_2025-01-03-22-14-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1979775.log +started roslaunch server http://158.130.50.37:40059/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1980640] +process[print_text-14]: started with pid [1980641] +process[rosbag_record-15]: started with pid [1980642] +started bagging! +[INFO] [1735960502.443262]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735960503.241616]: Stored episode 5. +[INFO] [1735960503.246807]: Saving video... +[INFO] [1735960503.660115]: Video saved! +[INFO] [1735960503.715957]: resetting environment +[INFO] [1735960503.728459]: cleared current plan +[INFO] [1735960503.728696]: moving to home +[INFO] [1735960507.434147]: reached home +[INFO] [1735960507.445208]: reset action history +[INFO] [1735960510.245337]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..eaa0937c80d4757f45258d88e388b831beda334c --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:10:43.296305", + "startedAt": "2025-01-04T03:10:42.882408", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.497125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.997, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.025, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 824.9600524902344 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e657ce96c9f4aab93050a3833e3008d2e580bbb4 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6345377befcb792e0c92371ad83ae363c65001343a244698a089ecd1cbae74c9", "size": 1148631, "path": "media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4"}, "global_step": 665, "_timestamp": 1735960519.5767248, "_runtime": 276.6860146522522, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "44ad851b779419a67be3aa1bb6defe2d621f28df4902cee5d6863f8824d36ed7", "size": 1140075, "path": "media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4"}, "eval/num_success": 4.0, "episode": 20.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "026bd92e4398e9ac5f9003d3135d96d78d16921a79b9bf6b022a5b6d473eeba3", "size": 1136822, "path": "media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "dcaa4b7b876ccb5c2cf6c134a59a92fe080cef90b4524c00540aae6779b8f695", "size": 1136449, "path": "media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f9ceef17a9c763063daba345a605c90e4fa65883e97b051371c35eae6c3d5c66", "size": 1132688, "path": "media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://0fq8xzl5avtc8do1p0evol14dhgrki3dc8npsxp9kfdv6cuwborjvqo3crlvmkhfmmdwj9qpia5akx48vy4vri4g25766tbbgfrsqnti3gp31w74ik0a0mtif162li55/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://x2k9pvm4t9gd3cjgjd7bjicihevx1or6lmf9xsourkr4b7e20kwbvsq9m2clrmznzqhcl18tz57w3gw7k71mboft7r90a5dqq8q7xrytgdbjmcrz5nsbxrrunswlwgrk:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 18, "total_num_episodes": 20, "total_success_rate": 0.9, "total_success_rate_lb_0.01": 0.6569254994078544, "total_success_rate_ub_0.01": 0.9899633600327942, "total_success_rate_lb_0.025": 0.7210156619106765, "total_success_rate_ub_0.025": 0.9872633720704922, "total_success_rate_lb_0.05": 0.7536698430269255, "total_success_rate_ub_0.05": 0.9710622577917205, "total_success_rate_lb_0.1": 0.7647808491654122, "total_success_rate_ub_0.1": 0.9711993922468375, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2e28e0b031c0bb400d35ae3f241a8f2e493191cff18a3dcf73d8f8d82ce9f898", "size": 505, "artifact_path": "wandb-client-artifact://7dnv70o1e61c6c6rla4i3mt91f8aytlf6as1v7p0vj2t35cm0mv6rwbjjgsf613aqqq4al7qdhno7i4u94zz653fzvpl9eh17rn1bwxaykbbiw3zt5ioqwrmshqsttca/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vy7jgn5ku9tyxyh4ulymaa9dag5nn7mhosawfiroamtg7jo5v9myce4pt2bh2a5s6uch220qvvxwtqgq7mf5t7e5ns4hlqb8yox6uisu2u6d1sbkm8tpbfspu8sm6lb4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 278}, "eval/total_success_rate": 0.8999999761581421, "eval/total_success_rate_lb_0.01": 0.6569254994392395, "eval/total_success_rate_ub_0.01": 0.9899633526802063, "eval/total_success_rate_lb_0.025": 0.7210156321525574, "eval/total_success_rate_ub_0.025": 0.9872633814811707, "eval/total_success_rate_lb_0.05": 0.7536698579788208, "eval/total_success_rate_ub_0.05": 0.9710622429847717, "eval/total_success_rate_lb_0.1": 0.7647808194160461, "eval/total_success_rate_ub_0.1": 0.9711993932723999} \ No newline at end of file diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug-internal.log b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e371b5618e9425bdbe723775b46891f446202b12 --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug-internal.log @@ -0,0 +1,786 @@ +2025-01-03 22:10:42,890 INFO StreamThr :1979941 [internal.py:wandb_internal():85] W&B internal server running at pid: 1979941, started at: 2025-01-03 22:10:42.890609 +2025-01-03 22:10:42,892 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:10:42,903 INFO WriterThread:1979941 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb +2025-01-03 22:10:42,903 DEBUG SenderThread:1979941 [sender.py:send():379] send: header +2025-01-03 22:10:42,911 DEBUG SenderThread:1979941 [sender.py:send():379] send: run +2025-01-03 22:10:43,184 INFO SenderThread:1979941 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files +2025-01-03 22:10:43,184 INFO SenderThread:1979941 [sender.py:_start_run_threads():1188] run started: 5q1nh8a1 with start time 1735960242.89071 +2025-01-03 22:10:43,200 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:10:43,201 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:10:43,278 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:10:43,282 DEBUG HandlerThread:1979941 [system_info.py:__init__():26] System info init +2025-01-03 22:10:43,282 DEBUG HandlerThread:1979941 [system_info.py:__init__():41] System info init done +2025-01-03 22:10:43,283 INFO HandlerThread:1979941 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:10:43,283 INFO SystemMonitor:1979941 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:10:43,283 INFO HandlerThread:1979941 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:10:43,283 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:10:43,284 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:10:43,285 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:10:43,285 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:10:43,286 INFO SystemMonitor:1979941 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:10:43,296 DEBUG HandlerThread:1979941 [system_info.py:probe():152] Probing system +2025-01-03 22:10:43,297 DEBUG HandlerThread:1979941 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:probe():200] Probing system done +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:10:43.296305', 'startedAt': '2025-01-04T03:10:42.882408', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.497125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 824.9600524902344}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:10:43,302 INFO HandlerThread:1979941 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:10:43,302 INFO HandlerThread:1979941 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:10:43,302 DEBUG HandlerThread:1979941 [system_info.py:_save_code():44] Saving code +2025-01-03 22:10:43,304 DEBUG HandlerThread:1979941 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:10:43,304 DEBUG HandlerThread:1979941 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:10:43,365 DEBUG HandlerThread:1979941 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:10:43,365 INFO HandlerThread:1979941 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:10:43,368 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:10:43,368 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:10:43,369 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:10:43,369 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:10:43,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:10:43,616 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:10:43,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:43,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:43,630 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/wh68wajm-diff.patch +2025-01-03 22:10:43,640 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/r76q07uj-code/FISH/eval_robot.py +2025-01-03 22:10:43,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:43,661 INFO HandlerThread:1979941 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/tb" + save: true +} +control { + relay_id: "d6dc34db7dbd4fd89138d7a373b283a2" +} +_info { + stream_id: "5q1nh8a1" +} + +2025-01-03 22:10:43,661 DEBUG HandlerThread:1979941 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:10:43,767 DEBUG SenderThread:1979941 [sender.py:send():379] send: telemetry +2025-01-03 22:10:43,767 DEBUG SenderThread:1979941 [sender.py:send():379] send: tbrecord +2025-01-03 22:10:43,768 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:10:43,768 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 with policy live +2025-01-03 22:10:43,956 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/p69umh6m-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:44,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code +2025-01-03 22:10:44,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH +2025-01-03 22:10:44,313 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/4wcn4b1s-wandb-metadata.json +2025-01-03 22:10:44,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:45,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:46,185 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:46,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:47,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:48,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:48,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:48,993 DEBUG SenderThread:1979941 [sender.py:send():379] send: config +2025-01-03 22:10:49,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:50,186 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:50,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:51,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:52,187 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:52,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:53,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:53,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:54,188 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:54,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:55,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:56,188 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:56,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:57,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:58,189 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:10:58,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:58,616 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:58,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:58,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:59,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:00,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:01,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:02,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:03,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:04,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:04,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:05,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:06,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:07,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:08,192 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:08,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:09,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:09,894 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:10,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:11,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:12,193 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:12,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:13,616 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:13,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:13,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:14,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:15,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:15,728 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:16,194 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:11:16,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:17,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:18,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:19,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:20,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:21,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:21,769 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:22,196 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:22,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:23,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:24,622 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:25,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:26,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:27,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:27,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:28,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:28,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:28,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:29,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:30,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:31,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:32,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:32,769 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:33,623 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:34,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:35,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:36,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:37,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:37,837 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:37,847 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:11:37,847 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 with policy now +2025-01-03 22:11:37,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:11:37,849 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:11:37,849 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:11:37,849 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:11:38,110 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0wstjb2o-media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos +2025-01-03 22:11:38,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:11:38,461 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/asfi8i93-media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:11:38,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:39,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:40,200 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:40,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:41,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:42,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:43,286 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:11:43,286 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:11:43,287 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:43,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:43,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:43,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:44,202 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:44,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:45,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:46,203 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:11:46,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:47,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:48,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:48,625 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:49,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:50,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:51,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:52,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:53,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:54,619 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:11:54,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:55,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:56,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:57,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:58,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:11:58,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:11:58,657 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:59,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:11:59,722 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:00,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:01,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:02,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:03,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:04,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:05,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:05,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:06,209 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:06,455 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/hrko8ho4-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:06,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:07,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:08,210 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:08,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:09,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:10,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:10,705 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:11,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:12,211 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:12,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:13,287 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:12:13,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:13,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:13,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:14,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:15,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:15,739 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:16,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:17,630 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:18,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:19,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:20,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:21,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:21,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:22,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:23,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:24,632 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:25,633 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:26,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:26,639 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:27,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:28,617 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:28,617 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:28,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:29,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:30,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:31,649 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:32,217 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:32,335 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:32,651 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:33,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:34,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:35,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:36,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:37,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:37,840 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:37,891 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:12:37,891 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 with policy now +2025-01-03 22:12:37,893 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:12:37,894 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:12:37,894 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:12:37,896 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:12:38,166 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/ztwsugy4-media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,219 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:12:38,479 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zcz0lrie-media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:12:38,659 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:39,660 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:40,220 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:40,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:41,660 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:42,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:43,287 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:12:43,288 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:43,626 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:43,626 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:43,669 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:44,221 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:44,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:45,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:46,222 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:46,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:47,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:48,319 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:48,662 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:49,082 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:12:49,082 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:12:49,083 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:12:49,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:12:49,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:12:49,663 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:50,223 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:50,665 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:51,663 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:52,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:53,325 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:53,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:54,224 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:12:54,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:55,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:56,666 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:57,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:58,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:12:58,618 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:12:58,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:12:58,751 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:12:59,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:00,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:01,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:02,227 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:02,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:03,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:04,624 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:04,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:05,667 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:06,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:07,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:08,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:09,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:10,057 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:10,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:11,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:12,230 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:12,668 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:13,288 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:13:13,618 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:13,619 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:13,671 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:14,669 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:15,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:15,680 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:16,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:17,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:18,670 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:19,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:20,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:20,335 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:13:20,335 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 with policy now +2025-01-03 22:13:20,338 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:13:20,338 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:13:20,338 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:13:20,339 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:13:20,607 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/kri8wm7e-media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:20,673 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:21,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:13:21,365 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:21,540 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/pf09ehnf-media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:13:21,675 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:22,233 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:22,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:23,686 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:24,679 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:25,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:26,235 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:26,677 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:26,962 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:27,826 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:28,236 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:28,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:28,628 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:28,819 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:29,816 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:30,818 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:31,823 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:32,785 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:32,835 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:33,834 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:34,837 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:35,829 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:36,830 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:37,832 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:38,633 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:38,834 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:39,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:40,839 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:41,840 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:42,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:43,288 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:13:43,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:43,620 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:43,711 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:43,836 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:44,839 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:45,841 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:46,844 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:47,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:47,932 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:13:47,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:13:47,933 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:13:48,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:13:48,484 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/w1jmyvd1-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:13:48,846 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:49,637 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:49,846 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:50,242 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:50,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:51,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:52,243 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:13:52,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:53,849 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:54,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:55,628 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:13:55,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:56,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:57,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:58,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:13:58,620 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:13:58,850 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:13:59,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:00,246 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:00,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:01,169 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:01,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:02,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:03,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:04,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:05,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:06,177 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:06,248 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:06,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:07,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:08,249 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:08,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:09,851 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:10,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:11,629 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:11,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:12,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:13,289 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:14:13,620 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:13,621 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:13,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:14,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:15,854 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:16,701 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:16,852 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:17,853 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:18,857 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:19,320 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:14:19,320 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 with policy now +2025-01-03 22:14:19,324 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:14:19,324 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:14:19,324 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:14:19,325 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:14:19,619 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zrlmijw0-media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:19,857 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:20,253 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:20,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:14:20,558 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/clgg49tw-media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:14:20,856 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:21,862 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:22,361 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:22,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:23,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:24,254 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:24,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:25,871 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:26,866 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:27,871 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:27,921 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:28,256 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:28,627 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:28,628 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:28,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:29,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:30,872 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:31,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:31,984 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:14:31,984 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:14:31,985 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:14:32,257 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:14:32,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:33,565 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:33,880 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:34,258 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:34,874 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:35,874 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:36,258 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:36,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:37,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:38,259 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:14:38,635 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:38,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:39,875 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:40,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:41,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:42,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:43,289 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:14:43,621 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:43,622 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:43,752 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:43,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:44,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:45,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:46,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:47,876 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:48,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:49,635 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:49,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:50,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:51,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:52,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:53,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:54,636 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:54,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:55,877 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:56,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:57,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:58,631 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:14:58,631 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:14:58,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:14:59,723 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:14:59,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:00,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:01,878 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:02,882 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:03,705 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:03,705 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 with policy now +2025-01-03 22:15:03,711 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:03,712 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:03,712 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:03,713 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:03,879 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:03,967 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0tw0g7tt-media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,268 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval +2025-01-03 22:15:04,536 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/0iiwb453-media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:04,729 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:04,883 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:05,887 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:06,269 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:06,900 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:07,897 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:08,269 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:08,887 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:09,888 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:10,248 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:10,892 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:11,897 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:12,271 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:12,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:13,290 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:15:13,641 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:15:13,641 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:15:13,901 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:14,905 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:15,797 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:15,903 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:16,896 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:17,899 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:18,273 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:18,542 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/qjabw0tk-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:18,674 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:15:18,674 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:15:18,815 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:18,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval +2025-01-03 22:15:19,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table +2025-01-03 22:15:19,450 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/yfrkg50a-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:19,542 INFO wandb-upload_0:1979941 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgdiy5yak +2025-01-03 22:15:19,653 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:15:19,661 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:15:19,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:19,924 INFO SenderThread:1979941 [sender.py:send_request_log_artifact():1525] logged artifact run-5q1nh8a1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2MDY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTE2NA==', 'latestArtifact': None}} +2025-01-03 22:15:19,924 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:19,924 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,925 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:19,925 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,929 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,929 DEBUG SenderThread:1979941 [sender.py:send():379] send: telemetry +2025-01-03 22:15:19,929 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,930 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,930 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,930 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,931 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,931 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,931 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,931 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,932 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,932 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,932 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,933 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,934 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,935 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,935 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,936 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,936 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:19,937 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:19,938 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:15:20,192 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/axlvm1rp-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:20,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:20,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval +2025-01-03 22:15:20,323 INFO wandb-upload_0:1979941 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_uhd4pk9 +2025-01-03 22:15:20,444 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/lmay8m1c-media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:20,737 INFO SenderThread:1979941 [sender.py:send_request_log_artifact():1525] logged artifact run-5q1nh8a1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk2MDcyMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NTE2Nw==', 'latestArtifact': None}} +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send():379] send: files +2025-01-03 22:15:20,737 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json with policy now +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:20,737 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:20,738 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:20,891 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:20,978 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/zttv0tae-media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:21,274 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:21,738 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:21,913 DEBUG SenderThread:1979941 [sender.py:send():379] send: exit +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():588] handling runtime: 278 +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_exit():594] send defer +2025-01-03 22:15:21,914 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,914 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:15:21,914 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,914 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:15:21,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:15:21,915 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:15:21,915 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:15:21,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:15:21,915 INFO HandlerThread:1979941 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:15:21,916 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:15:21,916 DEBUG SystemMonitor:1979941 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:15:21,916 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:15:21,916 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:15:21,967 INFO HandlerThread:1979941 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,968 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:15:21,968 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send():379] send: stats +2025-01-03 22:15:21,968 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,968 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:15:21,968 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:21,969 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:15:21,969 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:15:21,969 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:21,969 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:15:22,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:22,275 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:24,276 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:28,050 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:28,050 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:28,051 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,052 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:15:28,052 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send():379] send: history +2025-01-03 22:15:28,052 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:15:28,053 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,053 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:15:28,053 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:28,053 DEBUG SenderThread:1979941 [sender.py:send():379] send: summary +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:15:28,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:15:28,054 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,054 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:15:28,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,054 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:15:28,058 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:15:28,179 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:15:28,180 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:28,180 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:15:28,180 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:28,180 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:15:28,277 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:15:28,278 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:28,915 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:32,054 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:15:32,054 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:32,055 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:32,055 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:15:32,055 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:32,055 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:15:32,055 INFO SenderThread:1979941 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:15:32,057 INFO SenderThread:1979941 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:15:32,059 INFO SenderThread:1979941 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:15:32,069 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:15:32,069 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:32,069 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:15:32,069 DEBUG SenderThread:1979941 [sender.py:send():379] send: artifact +2025-01-03 22:15:32,279 INFO Thread-12 :1979941 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:32,377 INFO SenderThread:1979941 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'versionIndex': 411}}} +2025-01-03 22:15:32,378 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:32,378 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:15:32,378 INFO SenderThread:1979941 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:15:32,916 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml config.yaml +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json wandb-summary.json +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt requirements.txt +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:15:33,280 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:33,281 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log output.log +2025-01-03 22:15:33,281 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/diff.patch diff.patch +2025-01-03 22:15:33,285 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 media/videos/eval/4_eval_4_f9ceef17a9c763063dab.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 media/videos/eval/3_eval_3_dcaa4b7b876ccb5c2cf6.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 media/videos/eval/1_eval_1_44ad851b779419a67be3.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 media/videos/eval/2_eval_2_026bd92e4398e9ac5f90.mp4 +2025-01-03 22:15:33,286 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 media/videos/eval/0_eval_0_6345377befcb792e0c92.mp4 +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json media/table/eval/total_success_rate_ci_6_2e28e0b031c0bb400d35.table.json +2025-01-03 22:15:33,289 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:15:33,289 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,289 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,289 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:15:33,290 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,290 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:15:33,290 INFO SenderThread:1979941 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:15:33,461 INFO wandb-upload_2:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/requirements.txt +2025-01-03 22:15:33,468 INFO wandb-upload_1:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/config.yaml +2025-01-03 22:15:33,511 INFO wandb-upload_0:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/wandb-summary.json +2025-01-03 22:15:33,525 INFO wandb-upload_4:1979941 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/files/output.log +2025-01-03 22:15:33,573 INFO wandb-upload_3:1979941 [upload_job.py:push():130] Uploaded file /tmp/tmpe11hu3rowandb/cya0lkoz-events.out.tfevents.1735960243.leonmkim-ROG-Strix-G15CS-G15CS.1979775.0 +2025-01-03 22:15:33,773 INFO Thread-11 (_thread_body):1979941 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:15:33,773 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,773 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:15:33,773 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,773 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:15:33,773 INFO SenderThread:1979941 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:15:33,775 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,775 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:15:33,775 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:15:33,775 INFO SenderThread:1979941 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:15:33,843 INFO SenderThread:1979941 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:15:33,843 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:15:33,843 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,843 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send():379] send: final +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send():379] send: footer +2025-01-03 22:15:33,844 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:15:33,844 INFO HandlerThread:1979941 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:15:33,844 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: defer +2025-01-03 22:15:33,844 INFO SenderThread:1979941 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,845 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:15:33,845 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:15:33,846 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:15:33,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:15:33,847 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:15:33,848 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:15:33,898 DEBUG SenderThread:1979941 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:15:33,898 INFO MainThread:1979941 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:15:33,898 INFO MainThread:1979941 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:15:33,899 INFO MainThread:1979941 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:15:33,899 DEBUG HandlerThread:1979941 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:15:33,899 INFO HandlerThread:1979941 [handler.py:finish():882] shutting down handler +2025-01-03 22:15:34,847 INFO WriterThread:1979941 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [sender.py:finish():1615] shutting down sender +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:15:34,898 INFO SenderThread:1979941 [file_pusher.py:join():175] waiting for file pusher diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug.log b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bfb6f5a5592d45e2bb76c009a40119c59509418b --- /dev/null +++ b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Configure stats pid to 1979775 +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/settings +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug.log +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/logs/debug-internal.log +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:10:42,884 INFO MainThread:1979775 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1'} +2025-01-03 22:10:42,885 INFO MainThread:1979775 [wandb_init.py:init():619] starting backend +2025-01-03 22:10:42,885 INFO MainThread:1979775 [wandb_init.py:init():623] setting up manager +2025-01-03 22:10:42,889 INFO MainThread:1979775 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:10:42,890 INFO MainThread:1979775 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:10:42,902 INFO MainThread:1979775 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:10:42,909 INFO MainThread:1979775 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:10:43,200 INFO MainThread:1979775 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:10:43,270 INFO MainThread:1979775 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:10:43,270 INFO MainThread:1979775 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:10:43,617 INFO MainThread:1979775 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:10:43,619 INFO MainThread:1979775 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:10:43,619 INFO MainThread:1979775 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/5q1nh8a1/tb, True +2025-01-03 22:10:48,992 INFO MainThread:1979775 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:15:34,900 WARNING MsgRouterThr:1979775 [router.py:message_loop():77] message_loop has been closed diff --git a/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ed991405803dd90e7ee859f8e85a5b1c029d00d7 Binary files /dev/null and b/5q1nh8a1/wandb/run-20250103_221042-5q1nh8a1/run-5q1nh8a1.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38668e39565b344afcdd19c9ea3b3ab7325cf1ed --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 1 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1938_0' +wandb_notes: 1938_0_restarted_5 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/iss5qghl/.hydra/config.yaml b/iss5qghl/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/iss5qghl/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/iss5qghl/.hydra/hydra.yaml b/iss5qghl/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57a70cbb09dbe9ff06d24c3af1e2b140dfff56a7 --- /dev/null +++ b/iss5qghl/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/iss5qghl + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/iss5qghl/.hydra/overrides.yaml b/iss5qghl/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/iss5qghl/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/iss5qghl/eval_policy.log b/iss5qghl/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..6773ab755c799a7d091b848f0bce2eea69c73ed9 --- /dev/null +++ b/iss5qghl/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 21:18:54,739][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:18:54,743][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 21:18:59,020][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/n627ep2q/.hydra/hydra.yaml b/n627ep2q/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ac63b37e3badb655cb68ff7e6a7c0ea8546b4e6 --- /dev/null +++ b/n627ep2q/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/n627ep2q/episode_rosbags/aligned_depth_to_color_K.npy b/n627ep2q/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/n627ep2q/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/n627ep2q/episode_rosbags/cam_tf_world.npy b/n627ep2q/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/n627ep2q/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/n627ep2q/episode_rosbags/color_K.npy b/n627ep2q/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/n627ep2q/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/n627ep2q/episode_rosbags/depth_K.npy b/n627ep2q/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/n627ep2q/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/n627ep2q/episode_rosbags/episode_0_2025-01-03-22-40-23.bag b/n627ep2q/episode_rosbags/episode_0_2025-01-03-22-40-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..1cd034632ce81a9ba3a9a4bf0af664fe01c96e36 --- /dev/null +++ b/n627ep2q/episode_rosbags/episode_0_2025-01-03-22-40-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:836e182a5ff13aa4ed5027d3ed1507f86e49257c01fe801c1e5531a71b7ad0c5 +size 1416010277 diff --git a/n627ep2q/eval_robot.log b/n627ep2q/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bbdd8f0eb658b563000a2c35770faf0b28790b1c --- /dev/null +++ b/n627ep2q/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 22:39:31,785][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:39:31,788][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/n627ep2q/wandb/debug-internal.log b/n627ep2q/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f1ed123bbe7696670b4c04c56e879199f132af3d --- /dev/null +++ b/n627ep2q/wandb/debug-internal.log @@ -0,0 +1,795 @@ +2025-01-03 22:39:37,761 INFO StreamThr :1986224 [internal.py:wandb_internal():85] W&B internal server running at pid: 1986224, started at: 2025-01-03 22:39:37.760689 +2025-01-03 22:39:37,762 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:39:37,774 INFO WriterThread:1986224 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb +2025-01-03 22:39:37,775 DEBUG SenderThread:1986224 [sender.py:send():379] send: header +2025-01-03 22:39:37,782 DEBUG SenderThread:1986224 [sender.py:send():379] send: run +2025-01-03 22:39:37,997 INFO SenderThread:1986224 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files +2025-01-03 22:39:37,997 INFO SenderThread:1986224 [sender.py:_start_run_threads():1188] run started: n627ep2q with start time 1735961977.76136 +2025-01-03 22:39:38,013 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:39:38,013 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:39:38,084 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:39:38,089 DEBUG HandlerThread:1986224 [system_info.py:__init__():26] System info init +2025-01-03 22:39:38,089 DEBUG HandlerThread:1986224 [system_info.py:__init__():41] System info init done +2025-01-03 22:39:38,089 INFO HandlerThread:1986224 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:39:38,089 INFO SystemMonitor:1986224 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:39:38,089 INFO HandlerThread:1986224 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:39:38,090 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:39:38,091 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:39:38,091 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:39:38,092 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:39:38,092 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:39:38,103 DEBUG HandlerThread:1986224 [system_info.py:probe():152] Probing system +2025-01-03 22:39:38,104 DEBUG HandlerThread:1986224 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:probe():200] Probing system done +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:39:38.103396', 'startedAt': '2025-01-04T03:39:37.747674', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.999625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.096, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 857.3146286010742}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:39:38,109 INFO HandlerThread:1986224 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:39:38,109 INFO HandlerThread:1986224 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:_save_code():44] Saving code +2025-01-03 22:39:38,111 DEBUG HandlerThread:1986224 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:39:38,111 DEBUG HandlerThread:1986224 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:39:38,172 DEBUG HandlerThread:1986224 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:39:38,173 INFO HandlerThread:1986224 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:39:38,176 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:39:38,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:39:38,428 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:39:38,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:38,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:39:38,431 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:39:38,473 INFO HandlerThread:1986224 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/tb" + save: true +} +control { + relay_id: "7244c54b3bec4ada9efeadfa9a653d6e" +} +_info { + stream_id: "n627ep2q" +} + +2025-01-03 22:39:38,473 DEBUG HandlerThread:1986224 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:39:38,484 DEBUG SenderThread:1986224 [sender.py:send():379] send: telemetry +2025-01-03 22:39:38,523 DEBUG SenderThread:1986224 [sender.py:send():379] send: tbrecord +2025-01-03 22:39:38,524 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:39:38,524 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 with policy live +2025-01-03 22:39:38,571 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/wxme52vd-code/FISH/eval_robot.py +2025-01-03 22:39:38,613 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/n0ajzddu-diff.patch +2025-01-03 22:39:38,614 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/bwpmc7eo-wandb-metadata.json +2025-01-03 22:39:38,935 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/upiije4w-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:39:38,998 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code +2025-01-03 22:39:39,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:40,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:40,998 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:41,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:42,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:43,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:43,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:44,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:45,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:46,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:47,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:48,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:48,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:49,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:50,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:50,538 DEBUG SenderThread:1986224 [sender.py:send():379] send: config +2025-01-03 22:39:51,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:52,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:53,001 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:53,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:39:53,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:39:53,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:53,516 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:54,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:55,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:56,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:57,002 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:57,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:58,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:59,003 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:59,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:59,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:00,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:01,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:02,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:03,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:04,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:04,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:05,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:06,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:07,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:08,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:08,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:08,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:09,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:09,548 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:10,006 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:40:10,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:11,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:12,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:13,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:14,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:15,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:15,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:16,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:17,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:18,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:19,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:20,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:21,009 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:21,353 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:21,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:22,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:23,009 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:23,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:23,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:23,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:24,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:25,010 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:25,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:26,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:26,484 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:27,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:28,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:29,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:30,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:31,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:32,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:32,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:33,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:34,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:35,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:36,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:37,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:37,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:38,092 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:40:38,093 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:40:38,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:38,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:38,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:39,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:40,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:41,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:42,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:42,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:43,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:44,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:45,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:46,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:47,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:48,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:48,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:49,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:50,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:51,108 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:40:51,108 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 with policy now +2025-01-03 22:40:51,108 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:40:51,110 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:40:51,110 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:40:51,111 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:40:51,347 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/0f1fizq4-media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:51,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:40:52,019 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos +2025-01-03 22:40:52,296 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/tw8559up-media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:52,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:53,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:53,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:53,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:53,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:53,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:54,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:55,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:56,445 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:57,020 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:57,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:58,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:59,020 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:59,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:59,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:00,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:01,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:02,021 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:41:02,259 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/mtqkwm02-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:41:02,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:03,022 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:03,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:04,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:05,023 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:05,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:05,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:06,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:07,023 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:07,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:08,094 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:41:08,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:08,430 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:08,473 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:09,024 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:09,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:10,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:10,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:11,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:12,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:13,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:14,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:15,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:16,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:16,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:17,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:18,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:19,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:20,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:21,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:21,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:22,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:23,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:23,431 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:23,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:24,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:25,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:26,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:26,538 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:27,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:28,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:29,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:30,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:31,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:32,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:32,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:33,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:34,450 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:34,821 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:41:34,821 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 with policy now +2025-01-03 22:41:34,822 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:41:34,829 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:41:34,829 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:41:34,830 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:41:35,047 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/i0le1g1q-media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,300 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/pbvm3ae3-media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,455 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:36,449 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:37,034 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:37,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:37,846 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:38,094 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:41:38,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:38,433 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:38,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:39,454 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:40,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:41,036 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:41,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:42,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:43,036 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:43,277 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:43,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:44,456 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:45,456 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:46,457 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:47,458 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:48,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:48,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:49,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:50,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:51,458 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:52,471 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:53,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:53,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:53,439 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:53,481 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:54,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:55,463 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:56,462 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:57,466 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:58,467 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:59,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:59,462 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:00,467 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:01,472 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:02,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:03,468 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:04,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:04,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:05,468 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:06,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:07,472 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:08,095 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:42:08,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:08,436 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:08,481 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:09,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:09,529 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:10,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:11,474 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:12,475 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:13,046 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:13,284 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/c1wpsv05-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:13,474 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:13,922 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:13,922 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:13,922 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:14,046 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:14,475 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:14,922 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:15,047 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:15,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:16,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:17,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:18,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:19,048 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:19,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:20,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:20,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:21,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:22,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:23,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:23,432 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:23,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:24,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:25,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:25,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:26,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:27,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:28,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:29,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:30,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:31,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:31,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:32,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:33,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:34,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:35,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:36,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:36,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:37,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:38,095 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:42:38,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:38,432 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:38,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:39,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:40,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:41,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:41,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:42,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:43,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:44,480 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:45,387 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:42:45,387 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 with policy now +2025-01-03 22:42:45,394 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:42:45,394 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:45,394 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:45,394 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:45,480 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:45,597 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/24w6uv2d-media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:42:46,329 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/ycmn6zlr-media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,482 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:47,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:47,421 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:47,490 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:48,490 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:49,492 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:50,494 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:51,059 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:51,493 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:52,494 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:52,861 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:53,059 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:53,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:53,437 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:53,495 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:54,496 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:55,060 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:55,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:55,754 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:55,754 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:55,754 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:56,060 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:56,503 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:57,061 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:57,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:58,089 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:58,498 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:59,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:00,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:01,062 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:01,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:02,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:03,063 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:03,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:03,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:04,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:05,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:06,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:07,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:08,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:43:08,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:08,435 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:08,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:08,512 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:09,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:10,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:11,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:12,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:13,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:14,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:14,503 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:15,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:16,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:17,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:18,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:19,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:20,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:20,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:21,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:22,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:23,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:23,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:23,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:24,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:25,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:25,561 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:26,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:27,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:28,507 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:29,508 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:30,511 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:30,619 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:30,951 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:43:30,951 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 with policy now +2025-01-03 22:43:30,957 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:43:30,957 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:43:30,957 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:43:30,957 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:43:31,175 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/48r499yp-media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,464 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/pm0i7stl-media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:32,512 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:33,074 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:33,511 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:34,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:35,527 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:35,641 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:36,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:37,075 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:37,513 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:38,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:43:38,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:38,443 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:38,514 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:39,076 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:39,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:40,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:40,810 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:43:40,810 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:43:40,811 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:40,811 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:43:41,076 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:43:41,077 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:43:41,305 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/ht2mny2o-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:43:41,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:42,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:43,077 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:43,547 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:44,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:45,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:46,449 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:46,527 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:47,078 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:47,519 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:48,519 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:49,079 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:49,520 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:50,520 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:51,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:52,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:52,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:53,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:53,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:53,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:54,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:55,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:56,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:57,082 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:57,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:57,934 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:58,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:59,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:00,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:01,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:02,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:03,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:03,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:04,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:05,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:06,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:07,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:08,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:44:08,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:44:08,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:44:08,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:08,558 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:09,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:10,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:11,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:12,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:13,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:14,287 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:14,546 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:15,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:15,328 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:15,328 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f169542c93769083c885.mp4 with policy now +2025-01-03 22:44:15,329 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:15,329 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:15,329 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:15,330 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:15,540 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:15,593 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/jjjyrulg-media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:44:16,356 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/owwvr9zk-media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:17,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:17,530 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:18,538 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:19,533 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:20,025 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:20,548 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:21,091 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:21,535 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:22,535 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:22,859 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:22,860 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:22,862 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:23,091 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:23,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:23,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:23,451 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:44:23,451 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:44:23,551 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:23,805 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:44:23,805 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:44:23,937 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table +2025-01-03 22:44:24,277 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/d0ioxhcf-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:24,285 INFO wandb-upload_0:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4k6xd7lf +2025-01-03 22:44:24,365 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:44:24,375 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:24,533 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:24,691 INFO SenderThread:1986224 [sender.py:send_request_log_artifact():1525] logged artifact run-n627ep2q-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDE0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MDExNQ==', 'latestArtifact': None}} +2025-01-03 22:44:24,691 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:24,691 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:44:24,692 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:24,692 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,696 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,696 DEBUG SenderThread:1986224 [sender.py:send():379] send: telemetry +2025-01-03 22:44:24,697 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,697 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,697 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,717 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,718 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,718 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,718 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,741 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,742 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,742 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,742 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,765 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,765 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,776 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,777 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,791 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,791 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,806 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,806 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,807 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,807 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,819 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,819 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:44:24,957 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/gbbfvklb-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:25,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:25,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval +2025-01-03 22:44:25,177 INFO wandb-upload_2:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgb_crl40 +2025-01-03 22:44:25,311 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/7f40vtwc-media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:25,559 INFO SenderThread:1986224 [sender.py:send_request_log_artifact():1525] logged artifact run-n627ep2q-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDE1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MDExOQ==', 'latestArtifact': None}} +2025-01-03 22:44:25,559 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:25,559 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json with policy now +2025-01-03 22:44:25,559 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:25,560 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:25,560 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:25,560 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:25,561 DEBUG SenderThread:1986224 [sender.py:send():379] send: exit +2025-01-03 22:44:25,561 INFO SenderThread:1986224 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:44:25,561 INFO SenderThread:1986224 [sender.py:send_exit():588] handling runtime: 286 +2025-01-03 22:44:25,564 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_exit():594] send defer +2025-01-03 22:44:25,565 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,565 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:44:25,565 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:44:25,565 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,565 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:44:25,565 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:44:25,566 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:44:25,566 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,566 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:44:25,566 INFO HandlerThread:1986224 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:44:25,566 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:44:25,566 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:44:25,567 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:44:25,567 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:44:25,631 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:44:25,632 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:44:25,632 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:44:25,632 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,632 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:44:25,632 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:44:25,632 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:44:25,632 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,633 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:44:25,633 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,633 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:44:25,633 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:44:25,633 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,633 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:44:26,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:26,186 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/rir2dayx-media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:27,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:31,095 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,870 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:44:31,870 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:31,871 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,871 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:44:31,872 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:31,872 DEBUG SenderThread:1986224 [sender.py:send():379] send: summary +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:31,876 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:44:31,876 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,876 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:44:31,876 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:44:31,877 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:44:31,877 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:31,877 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,877 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:44:31,877 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,877 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:44:32,095 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:32,771 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:32,817 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:44:32,817 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:32,817 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:32,818 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:44:32,818 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:32,818 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:44:32,818 INFO SenderThread:1986224 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:44:32,820 INFO SenderThread:1986224 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:44:32,822 INFO SenderThread:1986224 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:44:32,837 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:44:32,837 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:32,837 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:44:32,837 DEBUG SenderThread:1986224 [sender.py:send():379] send: artifact +2025-01-03 22:44:33,096 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:33,251 INFO wandb-upload_1:1986224 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfc0zgqtm +2025-01-03 22:44:33,251 INFO wandb-upload_0:1986224 [upload_job.py:push():85] Skipped uploading /tmp/tmpx3m6664_/wandb-job.json +2025-01-03 22:44:33,364 INFO wandb-upload_2:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9ghtw20_ +2025-01-03 22:44:33,771 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:44:34,004 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:44:34,096 INFO SenderThread:1986224 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml config.yaml +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json wandb-summary.json +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt requirements.txt +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log output.log +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch diff.patch +2025-01-03 22:44:34,099 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:44:34,103 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,103 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,106 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:44:34,106 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,106 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:44:34,106 INFO SenderThread:1986224 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:44:34,265 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:34,277 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:44:34,304 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt +2025-01-03 22:44:34,331 INFO wandb-upload_4:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:34,391 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/bmpdbjn2-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:34,591 INFO Thread-11 (_thread_body):1986224 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:44:34,591 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,591 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:44:34,592 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,592 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:44:34,592 INFO SenderThread:1986224 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:44:34,593 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,593 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:44:34,593 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:44:34,772 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:44:34,811 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,811 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,811 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:44:34,811 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send():379] send: final +2025-01-03 22:44:34,812 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,812 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send():379] send: footer +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,812 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:44:34,812 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,813 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,813 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:44:34,813 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,813 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:44:34,815 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:44:34,816 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:44:34,816 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:34,865 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,865 INFO MainThread:1986224 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:44:34,865 INFO MainThread:1986224 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:44:34,866 INFO MainThread:1986224 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:44:34,866 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:44:34,866 INFO HandlerThread:1986224 [handler.py:finish():882] shutting down handler +2025-01-03 22:44:35,813 INFO WriterThread:1986224 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [sender.py:finish():1615] shutting down sender +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [file_pusher.py:join():175] waiting for file pusher diff --git a/n627ep2q/wandb/debug.log b/n627ep2q/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fa52514756cc31df4213527a428fa65eaa8db4d --- /dev/null +++ b/n627ep2q/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Configure stats pid to 1986035 +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/settings +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug.log +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug-internal.log +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q'} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():619] starting backend +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():623] setting up manager +2025-01-03 22:39:37,759 INFO MainThread:1986035 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:39:37,761 INFO MainThread:1986035 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:39:37,774 INFO MainThread:1986035 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:39:37,780 INFO MainThread:1986035 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:39:38,013 INFO MainThread:1986035 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:39:38,079 INFO MainThread:1986035 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:39:38,079 INFO MainThread:1986035 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:39:38,430 INFO MainThread:1986035 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:39:38,431 INFO MainThread:1986035 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:39:38,432 INFO MainThread:1986035 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/tb, True +2025-01-03 22:39:50,538 INFO MainThread:1986035 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:44:35,867 WARNING MsgRouterThr:1986035 [router.py:message_loop():77] message_loop has been closed diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54eebb93d7c809236f62d4fed8fec62fe0807621 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735961977 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3f45bbd23aa793167dd7ab04f8054b55af77b3c6 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[23, 25, 0.92, 0.7171168326897309, 0.9920074268191514, 0.771567851250694, 0.9898511803660591, 0.7991283684370691, 0.9769477172732132, 0.8087701200821364, 0.9770170402690408]]} \ No newline at end of file diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d00577c27415ae0705d1c25c21a781465997940 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1ff6c41d792e337b8f2fa6e45e5487696aaa8ac1327d66fbdb6f2b3f268ba6 +size 1192169 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b03372be3c1942c26c1bd350f248821ecb67b018 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcb69cc1c7293969286be93fceba12831456e52541fc39ea06bba60f47ef79d8 +size 1226892 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8911d97858d98545d94466889718f225367a67bd --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:528e4a4b9d24ffcf67ff9b2cad422ea9ebedc0fae9781bdfffc00dd2a80633a7 +size 1209710 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..327f73d3d964d0e5507ad1564869e75034298683 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e26516847c67094fe8eb83034b418e767ba00743a12e83d37a71c57e9100cd4a +size 1325955 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e7699e5d4197bb54c2844aec76b508b9dc99ee6 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f169542c93769083c88595cd6729c0d3b9ce8bf76f0863db22b2e8e952883514 +size 1294074 diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8540acb8eced405028cc6f4e21a6e07b9e6b6408 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log @@ -0,0 +1,261 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735961990.538824]: resetting environment +[INFO] [1735961990.541832]: cleared current plan +[INFO] [1735961990.543034]: moving to home +[INFO] [1735961991.545143]: reached home +[INFO] [1735961991.546373]: reset action history +[INFO] [1735961994.255249]: environment reset +Starting episode 0 +[INFO] [1735961994.257247]: resetting environment +[INFO] [1735961994.261344]: cleared current plan +[INFO] [1735961994.262272]: moving to home +[INFO] [1735961995.264444]: reached home +[INFO] [1735961995.266070]: reset action history +[INFO] [1735961997.974321]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735962019.306050]: resetting environment +[INFO] [1735962019.309136]: cleared current plan +[INFO] [1735962019.310037]: moving to home +[INFO] [1735962020.312430]: reached home +[INFO] [1735962020.314057]: reset action history +[INFO] [1735962023.025737]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/episode_rosbags/episode_0_2025-01-03-22-40-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1986035.log +started roslaunch server http://158.130.50.37:34487/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1986329] +process[print_text-2]: started with pid [1986353] +process[rosbag_record-3]: started with pid [1986354] +started bagging! +[INFO] [1735962050.147512]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735962050.886301]: Stored episode 1. +[INFO] [1735962050.886588]: Saving video... +[INFO] [1735962051.093719]: Video saved! +Starting episode 1 +[INFO] [1735962051.109473]: resetting environment +[INFO] [1735962051.116140]: cleared current plan +[INFO] [1735962051.116443]: moving to home +[INFO] [1735962054.720185]: reached home +[INFO] [1735962054.720629]: reset action history +[INFO] [1735962057.426372]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962061.867139]: resetting environment +[INFO] [1735962061.876627]: cleared current plan +[INFO] [1735962061.877246]: moving to home +[INFO] [1735962062.879390]: reached home +[INFO] [1735962062.879821]: reset action history +[INFO] [1735962065.586720]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/episode_rosbags/episode_1_2025-01-03-22-41-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1986035.log +started roslaunch server http://158.130.50.37:33159/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1986488] +process[print_text-5]: started with pid [1986489] +process[rosbag_record-6]: started with pid [1986490] +started bagging! +[INFO] [1735962093.785253]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735962094.575011]: Stored episode 2. +[INFO] [1735962094.578651]: Saving video... +[INFO] [1735962094.801763]: Video saved! +Starting episode 2 +[INFO] [1735962094.825914]: resetting environment +[INFO] [1735962094.845111]: cleared current plan +[INFO] [1735962094.845366]: moving to home +[INFO] [1735962098.453193]: reached home +[INFO] [1735962098.461615]: reset action history +[INFO] [1735962101.234876]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962132.799774]: resetting environment +[INFO] [1735962132.817745]: cleared current plan +[INFO] [1735962132.818893]: moving to home +[INFO] [1735962133.822017]: reached home +[INFO] [1735962133.831561]: reset action history +[INFO] [1735962136.608522]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/episode_rosbags/episode_2_2025-01-03-22-42-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1986035.log +started roslaunch server http://158.130.50.37:38561/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1986699] +process[print_text-8]: started with pid [1986700] +process[rosbag_record-9]: started with pid [1986701] +started bagging! +[INFO] [1735962164.351579]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735962165.138133]: Stored episode 3. +[INFO] [1735962165.143028]: Saving video... +[INFO] [1735962165.341463]: Video saved! +Starting episode 3 +[INFO] [1735962165.402443]: resetting environment +[INFO] [1735962165.420421]: cleared current plan +[INFO] [1735962165.420661]: moving to home +[INFO] [1735962169.028259]: reached home +[INFO] [1735962169.035563]: reset action history +[INFO] [1735962171.817816]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962175.004496]: resetting environment +[INFO] [1735962175.021138]: cleared current plan +[INFO] [1735962175.035599]: moving to home +[INFO] [1735962176.040944]: reached home +[INFO] [1735962176.045133]: reset action history +[INFO] [1735962178.998358]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/episode_rosbags/episode_3_2025-01-03-22-42-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1986035.log +started roslaunch server http://158.130.50.37:43341/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1986835] +process[print_text-11]: started with pid [1986836] +process[rosbag_record-12]: started with pid [1986837] +started bagging! +For topic gripper_width: timestamp difference is 105341899 for nearest: 1735962208528192746 - target: 1735962208422850847 at idx 249 +[INFO] [1735962209.751930]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735962210.618422]: Stored episode 4. +[INFO] [1735962210.629184]: Saving video... +[INFO] [1735962210.903229]: Video saved! +Starting episode 4 +[INFO] [1735962210.962057]: resetting environment +[INFO] [1735962210.984383]: cleared current plan +[INFO] [1735962210.984756]: moving to home +[INFO] [1735962214.593943]: reached home +[INFO] [1735962214.605620]: reset action history +[INFO] [1735962217.408326]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735962220.520999]: resetting environment +[INFO] [1735962220.538457]: cleared current plan +[INFO] [1735962220.548971]: moving to home +[INFO] [1735962221.563324]: reached home +[INFO] [1735962221.566769]: reset action history +[INFO] [1735962224.402572]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/episode_rosbags/episode_4_2025-01-03-22-43-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1986035.log +started roslaunch server http://158.130.50.37:42581/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1986982] +process[print_text-14]: started with pid [1986983] +process[rosbag_record-15]: started with pid [1986984] +started bagging! +For topic gripper_width: timestamp difference is -139225694 for nearest: 1735962235661550788 - target: 1735962235800776482 at idx 145 +[INFO] [1735962254.287086]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1735962255.047658]: Stored episode 5. +[INFO] [1735962255.056633]: Saving video... +[INFO] [1735962255.291169]: Video saved! +[INFO] [1735962255.329943]: resetting environment +[INFO] [1735962255.358017]: cleared current plan +[INFO] [1735962255.361404]: moving to home +[INFO] [1735962258.980494]: reached home +[INFO] [1735962258.995031]: reset action history +[INFO] [1735962261.848150]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..09dda0b5132842385b95008fa5edad1ec032d163 --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:39:38.103396", + "startedAt": "2025-01-04T03:39:37.747674", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.999625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.096, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 857.3146286010742 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..46f07409ecf5a89c6415cd6440721a10d30a098a --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7e1ff6c41d792e337b8f2fa6e45e5487696aaa8ac1327d66fbdb6f2b3f268ba6", "size": 1192169, "path": "media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4"}, "global_step": 665, "_timestamp": 1735962264.2340288, "_runtime": 286.4726688861847, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "dcb69cc1c7293969286be93fceba12831456e52541fc39ea06bba60f47ef79d8", "size": 1226892, "path": "media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4"}, "eval/num_success": 5.0, "episode": 25.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "528e4a4b9d24ffcf67ff9b2cad422ea9ebedc0fae9781bdfffc00dd2a80633a7", "size": 1209710, "path": "media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e26516847c67094fe8eb83034b418e767ba00743a12e83d37a71c57e9100cd4a", "size": 1325955, "path": "media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "f169542c93769083c88595cd6729c0d3b9ce8bf76f0863db22b2e8e952883514", "size": 1294074, "path": "media/videos/eval/4_eval_4_f169542c93769083c885.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://b9gqomxtyd043vk90dxpygqdmy9z8xy3cblgmar1axuq0wtk5gszap2yiypk32augd0jhoma9b0ikaajc4ij8ci8g5uns42xhjol03k0rz98cmlpv66jjrymz2dgm7it/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://kmagfppwjsa9qmk7kaxaakg7zn84up7pyvoybujeoca1pr902ummn19nig8hgm7a83tagchjcc7dt1rcndbk2ooe2cgvziy5643k3b3o1iohwc8olkax7dallx7idlw6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 23, "total_num_episodes": 25, "total_success_rate": 0.92, "total_success_rate_lb_0.01": 0.7171168326897309, "total_success_rate_ub_0.01": 0.9920074268191514, "total_success_rate_lb_0.025": 0.771567851250694, "total_success_rate_ub_0.025": 0.9898511803660591, "total_success_rate_lb_0.05": 0.7991283684370691, "total_success_rate_ub_0.05": 0.9769477172732132, "total_success_rate_lb_0.1": 0.8087701200821364, "total_success_rate_ub_0.1": 0.9770170402690408, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "5669fc74eb145fa070eedbaee87953dadc0dc2c311020c4e4438136f5e9f21bd", "size": 505, "artifact_path": "wandb-client-artifact://ljopitjd5y16bsu9x4p54s9txuzw19ouf62m4t82obk8oep1924r4efyy890l611lgigg9nydtauaftp33f04e634zv5zuzl78g6ucndws9twelacqfvh5bnswhbsln3/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://5jeyh5lucxne5342a34umkv1tmorlsd7i8j6j3vesmv4r7bbfaxlk463li65zld5x8j1zfnngww7sqxia68n52ik40sopsykfjq4gvsb445hhz4ybu1e08sqyyc7gp1e:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 286}, "eval/total_success_rate": 0.9200000166893005, "eval/total_success_rate_lb_0.01": 0.7171168327331543, "eval/total_success_rate_ub_0.01": 0.9920074343681335, "eval/total_success_rate_lb_0.025": 0.7715678215026855, "eval/total_success_rate_ub_0.025": 0.989851176738739, "eval/total_success_rate_lb_0.05": 0.7991283535957336, "eval/total_success_rate_ub_0.05": 0.9769477248191833, "eval/total_success_rate_lb_0.1": 0.8087701201438904, "eval/total_success_rate_ub_0.1": 0.9770170450210571} \ No newline at end of file diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug-internal.log b/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f1ed123bbe7696670b4c04c56e879199f132af3d --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug-internal.log @@ -0,0 +1,795 @@ +2025-01-03 22:39:37,761 INFO StreamThr :1986224 [internal.py:wandb_internal():85] W&B internal server running at pid: 1986224, started at: 2025-01-03 22:39:37.760689 +2025-01-03 22:39:37,762 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:39:37,774 INFO WriterThread:1986224 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb +2025-01-03 22:39:37,775 DEBUG SenderThread:1986224 [sender.py:send():379] send: header +2025-01-03 22:39:37,782 DEBUG SenderThread:1986224 [sender.py:send():379] send: run +2025-01-03 22:39:37,997 INFO SenderThread:1986224 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files +2025-01-03 22:39:37,997 INFO SenderThread:1986224 [sender.py:_start_run_threads():1188] run started: n627ep2q with start time 1735961977.76136 +2025-01-03 22:39:38,013 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:39:38,013 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:39:38,084 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:39:38,089 DEBUG HandlerThread:1986224 [system_info.py:__init__():26] System info init +2025-01-03 22:39:38,089 DEBUG HandlerThread:1986224 [system_info.py:__init__():41] System info init done +2025-01-03 22:39:38,089 INFO HandlerThread:1986224 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:39:38,089 INFO SystemMonitor:1986224 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:39:38,089 INFO HandlerThread:1986224 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:39:38,090 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:39:38,091 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:39:38,091 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:39:38,092 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:39:38,092 INFO SystemMonitor:1986224 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:39:38,103 DEBUG HandlerThread:1986224 [system_info.py:probe():152] Probing system +2025-01-03 22:39:38,104 DEBUG HandlerThread:1986224 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:probe():200] Probing system done +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:39:38.103396', 'startedAt': '2025-01-04T03:39:37.747674', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.999625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.096, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 857.3146286010742}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:39:38,109 INFO HandlerThread:1986224 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:39:38,109 INFO HandlerThread:1986224 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:39:38,109 DEBUG HandlerThread:1986224 [system_info.py:_save_code():44] Saving code +2025-01-03 22:39:38,111 DEBUG HandlerThread:1986224 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:39:38,111 DEBUG HandlerThread:1986224 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:39:38,172 DEBUG HandlerThread:1986224 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:39:38,173 INFO HandlerThread:1986224 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:39:38,176 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:39:38,176 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:39:38,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:39:38,428 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:39:38,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:38,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:39:38,431 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:39:38,473 INFO HandlerThread:1986224 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/tb" + save: true +} +control { + relay_id: "7244c54b3bec4ada9efeadfa9a653d6e" +} +_info { + stream_id: "n627ep2q" +} + +2025-01-03 22:39:38,473 DEBUG HandlerThread:1986224 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:39:38,484 DEBUG SenderThread:1986224 [sender.py:send():379] send: telemetry +2025-01-03 22:39:38,523 DEBUG SenderThread:1986224 [sender.py:send():379] send: tbrecord +2025-01-03 22:39:38,524 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:39:38,524 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 with policy live +2025-01-03 22:39:38,571 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/wxme52vd-code/FISH/eval_robot.py +2025-01-03 22:39:38,613 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/n0ajzddu-diff.patch +2025-01-03 22:39:38,614 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/bwpmc7eo-wandb-metadata.json +2025-01-03 22:39:38,935 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/upiije4w-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:39:38,998 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH +2025-01-03 22:39:38,999 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code +2025-01-03 22:39:39,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:40,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:40,998 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:41,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:42,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:43,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:43,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:44,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:45,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:46,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:47,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:48,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:48,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:49,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:50,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:50,538 DEBUG SenderThread:1986224 [sender.py:send():379] send: config +2025-01-03 22:39:51,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:52,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:53,001 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:53,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:39:53,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:39:53,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:53,516 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:54,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:55,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:56,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:57,002 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:57,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:58,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:39:59,003 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:39:59,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:39:59,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:00,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:01,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:02,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:03,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:04,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:04,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:05,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:06,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:07,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:08,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:08,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:08,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:09,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:09,548 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:10,006 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:40:10,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:11,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:12,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:13,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:14,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:15,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:15,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:16,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:17,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:18,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:19,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:20,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:21,009 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:21,353 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:21,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:22,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:23,009 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:23,428 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:23,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:23,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:24,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:25,010 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:25,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:26,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:26,484 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:27,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:28,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:29,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:30,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:31,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:32,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:32,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:33,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:34,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:35,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:36,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:37,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:37,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:38,092 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:40:38,093 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:40:38,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:38,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:38,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:39,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:40,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:41,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:42,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:42,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:43,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:44,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:45,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:46,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:47,438 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:48,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:48,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:49,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:50,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:51,108 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:40:51,108 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 with policy now +2025-01-03 22:40:51,108 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:40:51,110 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:40:51,110 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:40:51,111 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:40:51,347 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/0f1fizq4-media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:51,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media +2025-01-03 22:40:52,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:40:52,019 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos +2025-01-03 22:40:52,296 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/tw8559up-media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:40:52,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:53,018 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:53,429 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:40:53,429 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:40:53,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:53,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:54,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:55,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:56,445 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:57,020 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:57,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:58,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:40:59,020 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:40:59,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:40:59,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:00,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:01,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:02,021 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:41:02,259 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/mtqkwm02-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:41:02,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:03,022 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:03,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:04,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:05,023 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:05,431 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:05,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:06,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:07,023 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:07,441 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:08,094 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:41:08,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:08,430 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:08,473 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:09,024 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:09,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:10,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:10,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:11,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:12,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:13,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:14,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:15,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:16,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:16,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:17,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:18,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:19,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:20,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:21,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:21,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:22,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:23,430 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:23,431 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:23,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:24,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:25,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:26,446 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:26,538 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:27,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:28,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:29,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:30,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:31,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:32,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:32,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:33,447 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:34,450 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:34,821 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:41:34,821 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 with policy now +2025-01-03 22:41:34,822 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:41:34,829 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:41:34,829 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:41:34,830 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,033 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:41:35,047 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/i0le1g1q-media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,300 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/pbvm3ae3-media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:41:35,455 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:36,449 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:37,034 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:37,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:37,846 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:38,094 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:41:38,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:38,433 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:38,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:39,454 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:40,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:41,036 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:41,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:42,453 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:43,036 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:41:43,277 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:43,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:44,456 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:45,456 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:46,457 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:47,458 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:48,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:48,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:49,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:50,459 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:51,458 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:52,471 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:53,437 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:53,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:41:53,439 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:41:53,481 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:54,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:55,463 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:56,462 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:57,466 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:58,467 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:41:59,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:41:59,462 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:00,467 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:01,472 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:02,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:03,468 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:04,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:04,465 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:05,468 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:06,469 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:07,472 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:08,095 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:42:08,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:08,436 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:08,481 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:09,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:09,529 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:10,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:11,474 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:12,475 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:13,046 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:13,284 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/c1wpsv05-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:13,474 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:13,922 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:13,922 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:13,922 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:14,046 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:14,475 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:14,922 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:15,047 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:15,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:16,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:17,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:18,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:19,048 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:19,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:20,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:20,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:21,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:22,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:23,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:23,432 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:23,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:24,476 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:25,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:25,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:26,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:27,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:28,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:29,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:30,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:31,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:31,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:32,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:33,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:34,477 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:35,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:36,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:36,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:37,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:38,095 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:42:38,432 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:38,432 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:38,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:39,478 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:40,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:41,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:41,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:42,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:43,479 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:44,480 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:45,387 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:42:45,387 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 with policy now +2025-01-03 22:42:45,394 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:42:45,394 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:45,394 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:45,394 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:45,480 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:45,597 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/24w6uv2d-media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:42:46,329 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/ycmn6zlr-media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:42:46,482 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:47,057 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:47,421 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:47,490 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:48,490 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:49,492 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:50,494 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:51,059 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:51,493 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:52,494 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:52,861 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:53,059 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:53,436 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:42:53,437 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:42:53,495 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:54,496 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:55,060 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:42:55,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:55,754 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:42:55,754 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:42:55,754 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:42:56,060 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:42:56,503 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:57,061 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:42:57,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:58,089 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:42:58,498 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:42:59,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:00,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:01,062 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:01,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:02,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:03,063 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:03,439 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:03,499 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:04,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:05,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:06,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:07,500 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:08,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:43:08,435 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:08,435 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:08,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:08,512 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:09,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:10,501 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:11,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:12,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:13,502 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:14,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:14,503 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:15,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:16,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:17,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:18,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:19,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:20,440 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:20,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:21,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:22,504 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:23,433 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:23,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:23,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:24,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:25,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:25,561 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:26,505 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:27,506 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:28,507 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:29,508 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:30,511 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:30,619 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:30,951 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:43:30,951 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 with policy now +2025-01-03 22:43:30,957 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:43:30,957 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:43:30,957 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:43:30,957 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,073 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:43:31,175 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/48r499yp-media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,464 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/pm0i7stl-media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:43:31,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:32,512 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:33,074 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:33,511 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:34,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:35,527 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:35,641 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:36,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:37,075 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:37,513 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:38,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:43:38,442 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:38,443 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:38,514 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:39,076 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:39,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:40,526 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:40,810 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:43:40,810 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:43:40,811 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:40,811 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:43:41,076 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:43:41,077 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:43:41,305 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/ht2mny2o-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:43:41,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:42,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:43,077 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:43,547 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:44,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:45,528 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:46,449 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:46,527 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:47,078 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:47,519 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:48,519 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:49,079 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:49,520 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:50,520 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:51,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:52,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:52,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:53,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:43:53,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:43:53,521 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:54,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:55,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:56,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:57,082 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:43:57,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:57,934 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:43:58,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:43:59,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:00,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:01,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:02,522 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:03,443 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:03,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:04,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:05,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:06,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:07,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:08,096 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:44:08,434 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:44:08,434 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:44:08,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:08,558 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:09,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:10,523 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:11,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:12,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:13,524 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:14,287 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:14,546 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:15,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:15,328 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:15,328 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_f169542c93769083c885.mp4 with policy now +2025-01-03 22:44:15,329 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:15,329 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:15,329 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:15,330 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:15,540 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:15,593 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/jjjyrulg-media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval +2025-01-03 22:44:16,356 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/owwvr9zk-media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:16,542 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:17,089 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:17,530 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:18,538 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:19,533 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:20,025 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:20,548 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:21,091 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:21,535 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:22,535 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:22,859 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:22,860 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:22,862 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:23,091 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:23,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:23,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:23,451 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:44:23,451 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:44:23,551 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:23,805 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:44:23,805 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:44:23,937 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval +2025-01-03 22:44:24,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table +2025-01-03 22:44:24,277 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/d0ioxhcf-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:24,285 INFO wandb-upload_0:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4k6xd7lf +2025-01-03 22:44:24,365 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:44:24,375 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:44:24,533 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:24,691 INFO SenderThread:1986224 [sender.py:send_request_log_artifact():1525] logged artifact run-n627ep2q-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDE0NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MDExNQ==', 'latestArtifact': None}} +2025-01-03 22:44:24,691 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:24,691 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:44:24,692 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:24,692 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,696 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,696 DEBUG SenderThread:1986224 [sender.py:send():379] send: telemetry +2025-01-03 22:44:24,697 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,697 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,697 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,717 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,718 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,718 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,718 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,741 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,742 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,742 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,742 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,765 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,765 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,776 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,777 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,791 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,791 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,806 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,806 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,807 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,807 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:24,819 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:24,819 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:44:24,957 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/gbbfvklb-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:25,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:25,092 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:25,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval +2025-01-03 22:44:25,177 INFO wandb-upload_2:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgb_crl40 +2025-01-03 22:44:25,311 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/7f40vtwc-media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:25,559 INFO SenderThread:1986224 [sender.py:send_request_log_artifact():1525] logged artifact run-n627ep2q-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDE1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5MDExOQ==', 'latestArtifact': None}} +2025-01-03 22:44:25,559 DEBUG SenderThread:1986224 [sender.py:send():379] send: files +2025-01-03 22:44:25,559 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json with policy now +2025-01-03 22:44:25,559 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:25,560 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:25,560 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:25,560 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:25,561 DEBUG SenderThread:1986224 [sender.py:send():379] send: exit +2025-01-03 22:44:25,561 INFO SenderThread:1986224 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:44:25,561 INFO SenderThread:1986224 [sender.py:send_exit():588] handling runtime: 286 +2025-01-03 22:44:25,564 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_exit():594] send defer +2025-01-03 22:44:25,565 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,565 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:44:25,565 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:44:25,565 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,565 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:44:25,565 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,565 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:44:25,566 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:44:25,566 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,566 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:44:25,566 INFO HandlerThread:1986224 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:44:25,566 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:44:25,566 DEBUG SystemMonitor:1986224 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:44:25,567 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:44:25,567 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:44:25,631 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:44:25,632 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:44:25,632 INFO HandlerThread:1986224 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:44:25,632 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,632 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:44:25,632 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:44:25,632 DEBUG SenderThread:1986224 [sender.py:send():379] send: stats +2025-01-03 22:44:25,632 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,633 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:44:25,633 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:25,633 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:44:25,633 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:44:25,633 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:25,633 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:44:26,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:26,186 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/rir2dayx-media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:27,093 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:31,095 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:31,869 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,870 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:44:31,870 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send():379] send: history +2025-01-03 22:44:31,870 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:44:31,871 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,871 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:44:31,872 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:31,872 DEBUG SenderThread:1986224 [sender.py:send():379] send: summary +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:44:31,876 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:44:31,876 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,876 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:44:31,876 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,876 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:44:31,877 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:44:31,877 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:44:31,877 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:31,877 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:44:31,877 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:31,877 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:44:32,095 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:32,771 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:32,817 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:44:32,817 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:32,817 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:32,818 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:44:32,818 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:32,818 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:44:32,818 INFO SenderThread:1986224 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:44:32,820 INFO SenderThread:1986224 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:44:32,822 INFO SenderThread:1986224 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:44:32,837 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:44:32,837 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:32,837 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:44:32,837 DEBUG SenderThread:1986224 [sender.py:send():379] send: artifact +2025-01-03 22:44:33,096 INFO Thread-12 :1986224 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:33,251 INFO wandb-upload_1:1986224 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfc0zgqtm +2025-01-03 22:44:33,251 INFO wandb-upload_0:1986224 [upload_job.py:push():85] Skipped uploading /tmp/tmpx3m6664_/wandb-job.json +2025-01-03 22:44:33,364 INFO wandb-upload_2:1986224 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9ghtw20_ +2025-01-03 22:44:33,771 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk5MDI2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzk2NjYyMA==', 'versionIndex': 412}}} +2025-01-03 22:44:34,004 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:44:34,004 INFO SenderThread:1986224 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:44:34,096 INFO SenderThread:1986224 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml config.yaml +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json wandb-summary.json +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt requirements.txt +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log output.log +2025-01-03 22:44:34,097 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/diff.patch diff.patch +2025-01-03 22:44:34,099 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 media/videos/eval/1_eval_1_dcb69cc1c7293969286b.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/4_eval_4_f169542c93769083c885.mp4 media/videos/eval/4_eval_4_f169542c93769083c885.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 media/videos/eval/2_eval_2_528e4a4b9d24ffcf67ff.mp4 +2025-01-03 22:44:34,102 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 media/videos/eval/3_eval_3_e26516847c67094fe8eb.mp4 +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 media/videos/eval/0_eval_0_7e1ff6c41d792e337b8f.mp4 +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json media/table/eval/total_success_rate_ci_6_5669fc74eb145fa070ee.table.json +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:44:34,103 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:44:34,103 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,103 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,106 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:44:34,106 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,106 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:44:34,106 INFO SenderThread:1986224 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:44:34,265 INFO wandb-upload_0:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/wandb-summary.json +2025-01-03 22:44:34,277 INFO wandb-upload_3:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/config.yaml +2025-01-03 22:44:34,304 INFO wandb-upload_2:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/requirements.txt +2025-01-03 22:44:34,331 INFO wandb-upload_4:1986224 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/files/output.log +2025-01-03 22:44:34,391 INFO wandb-upload_1:1986224 [upload_job.py:push():130] Uploaded file /tmp/tmp1jofv5wlwandb/bmpdbjn2-events.out.tfevents.1735961978.leonmkim-ROG-Strix-G15CS-G15CS.1986035.0 +2025-01-03 22:44:34,591 INFO Thread-11 (_thread_body):1986224 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:44:34,591 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,591 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:44:34,592 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,592 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:44:34,592 INFO SenderThread:1986224 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:44:34,593 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,593 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:44:34,593 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:44:34,593 INFO SenderThread:1986224 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:44:34,772 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:44:34,811 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,811 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,811 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:44:34,811 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:44:34,811 INFO SenderThread:1986224 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send():379] send: final +2025-01-03 22:44:34,812 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:44:34,812 INFO HandlerThread:1986224 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send():379] send: footer +2025-01-03 22:44:34,812 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: defer +2025-01-03 22:44:34,812 INFO SenderThread:1986224 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:44:34,812 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,813 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,813 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:44:34,813 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:44:34,813 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:44:34,815 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:44:34,816 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:44:34,816 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:44:34,865 DEBUG SenderThread:1986224 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:44:34,865 INFO MainThread:1986224 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:44:34,865 INFO MainThread:1986224 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:44:34,866 INFO MainThread:1986224 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:44:34,866 DEBUG HandlerThread:1986224 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:44:34,866 INFO HandlerThread:1986224 [handler.py:finish():882] shutting down handler +2025-01-03 22:44:35,813 INFO WriterThread:1986224 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [sender.py:finish():1615] shutting down sender +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:44:35,865 INFO SenderThread:1986224 [file_pusher.py:join():175] waiting for file pusher diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug.log b/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fa52514756cc31df4213527a428fa65eaa8db4d --- /dev/null +++ b/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Configure stats pid to 1986035 +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/settings +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug.log +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/wandb/run-20250103_223937-n627ep2q/logs/debug-internal.log +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q'} +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():619] starting backend +2025-01-03 22:39:37,753 INFO MainThread:1986035 [wandb_init.py:init():623] setting up manager +2025-01-03 22:39:37,759 INFO MainThread:1986035 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:39:37,761 INFO MainThread:1986035 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:39:37,774 INFO MainThread:1986035 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:39:37,780 INFO MainThread:1986035 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:39:38,013 INFO MainThread:1986035 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:39:38,079 INFO MainThread:1986035 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:39:38,079 INFO MainThread:1986035 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:39:38,429 INFO MainThread:1986035 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:39:38,430 INFO MainThread:1986035 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:39:38,431 INFO MainThread:1986035 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:39:38,432 INFO MainThread:1986035 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/n627ep2q/tb, True +2025-01-03 22:39:50,538 INFO MainThread:1986035 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 22:44:35,867 WARNING MsgRouterThr:1986035 [router.py:message_loop():77] message_loop has been closed diff --git a/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb b/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb new file mode 100644 index 0000000000000000000000000000000000000000..273c49260492bbbf48fd8c04c6a909362070f7ab Binary files /dev/null and b/n627ep2q/wandb/run-20250103_223937-n627ep2q/run-n627ep2q.wandb differ diff --git a/snapshot_10013.pt b/snapshot_10013.pt new file mode 100644 index 0000000000000000000000000000000000000000..79e732b18b9e14f269ffda9355738e682e3dca6d --- /dev/null +++ b/snapshot_10013.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf27537bf7f5719a73849655471ff3dc41b2e2f4c89753b9c03a2b9347b691b +size 910102870 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..126677af13f4d9a1ae50c94715d9cbdc3b1d4353 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e45c87c56d552ee4253084aa5405e67862bad2a8a870adde241f6fad73927ce +size 910102870 diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf90634dfd198ab0d4c9e43fc7f6ab1b665fec75 --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3087c6ce0317fd28b108837abcb3187c12caa37d48699d535cb0c04ea26c68b +size 910102870 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..699e24e71314e4fb167befa00335816d478becac --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e62af0182020f30c2314362227b29ff505cc135f28614555b85db4aaeb50299 +size 910102870 diff --git a/snapshot_12040.pt b/snapshot_12040.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7969697dc59b2e39d2ec1f676c67a7c9c454887 --- /dev/null +++ b/snapshot_12040.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5f09e0a8ae8888386484351b624dfd202eb0ecfb5b6632b6948267330905da4 +size 910102870 diff --git a/snapshot_12749.pt b/snapshot_12749.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d4ed1929d3f685bfbe2217e62f77e071942f34f --- /dev/null +++ b/snapshot_12749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39c20b530787ea7e0ab4e2b3eb4f27df7ed7a0bb4a028b2d7e1172fce48342ca +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e32c09f4ecdb3d8b67a040cc40a38ef26900571 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89acb138791677609ac47bf9c46864475499a8f39a56e91b09a3d427bd543a09 +size 910102870 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ad57853d6390e6d281e2e5aa24e9c588bedb97e --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6492a29b9481e1f498d3a2b3243d00905930a813301e2b450e13de72226f675a +size 910102870 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..95a26c22365c8e27547fb90a9e0779d4bfe83520 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd2421394442e59eb79a6b0796468c84e1f19cc8287b29b1d92470fbd561f2d4 +size 910102360 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..6daccefe838ba157edd770357d6e370b8d94b3d1 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9afad3c416569cd80a820e71b6dcdc6dfadc88ac054d47252f1edac0780d106 +size 910102870 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..24345b440477b8147bb7f06c04b69382758e5ee4 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb7a3b4bf2f5182483de5e13523efb4d65a9abb93eb4d40b39cdab2be56ce9c9 +size 910102870 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4698875e9d3c18cc6b99512646ce3022080b7684 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3a1ed2cca56eb7cad4655c751006ac797578f3d11c9fb1a8b8223fa4b4ffed +size 910102870 diff --git a/snapshot_8499.pt b/snapshot_8499.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d74af1a9be116e05e071cba97aac8089769fa3 --- /dev/null +++ b/snapshot_8499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a6c1ec95453e45ddf76e5af269e462b0b5d83912ef2dc92de1da1d51ec9f399 +size 910102870 diff --git a/snapshot_8749.pt b/snapshot_8749.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b105a55b09c416f15c84a9e347d543d4cf22206 --- /dev/null +++ b/snapshot_8749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866f198eb223f5e7c6904daa1e4b029b73dc616a9a9f1b3dd7236d3fb6b4edf5 +size 910102870 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb544d8a35813278396080307426a7f948b34005 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737f3e7603d6f9868bdd52530fd98b4a508025124b8237d72bd13819abf004ac +size 910102870 diff --git a/snapshot_9749.pt b/snapshot_9749.pt new file mode 100644 index 0000000000000000000000000000000000000000..6240c321678d79c0fa9658ba94e75b737cd5764c --- /dev/null +++ b/snapshot_9749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5be297a298659c63b0002b3cc0559e3b1bdc879bf0035f69c911644a22011b +size 910102870 diff --git a/snapshot_9999.pt b/snapshot_9999.pt new file mode 100644 index 0000000000000000000000000000000000000000..79e732b18b9e14f269ffda9355738e682e3dca6d --- /dev/null +++ b/snapshot_9999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf27537bf7f5719a73849655471ff3dc41b2e2f4c89753b9c03a2b9347b691b +size 910102870 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ad57853d6390e6d281e2e5aa24e9c588bedb97e --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6492a29b9481e1f498d3a2b3243d00905930a813301e2b450e13de72226f675a +size 910102870