diff --git a/.gitattributes b/.gitattributes index 394fc495e2f262c52be3ea9c7bba89c6832eec1d..8dfa387ebd3e7509bf07d09a4b232b9e4cfc6a27 100644 --- a/.gitattributes +++ b/.gitattributes @@ -234,3 +234,47 @@ kccehv5k/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text kccehv5k/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text kccehv5k/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_2_2025-01-07-21-51-32.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_13_2025-01-07-22-00-06.bag filter=lfs diff=lfs merge=lfs -text +3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_10_2025-01-07-21-57-39.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_7_2025-01-07-21-55-24.bag filter=lfs diff=lfs merge=lfs -text +3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/episode_rosbags/episode_2_2025-01-06-17-15-32.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_0_2025-01-07-21-50-00.bag filter=lfs diff=lfs merge=lfs -text +3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +3d3i07nt/episode_rosbags/episode_4_2025-01-06-21-04-27.bag filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/episode_rosbags/episode_2_2025-01-06-17-09-45.bag filter=lfs diff=lfs merge=lfs -text +3d3i07nt/episode_rosbags/episode_3_2025-01-06-21-03-44.bag filter=lfs diff=lfs merge=lfs -text +3d3i07nt/episode_rosbags/episode_1_2025-01-06-21-01-55.bag filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/episode_rosbags/episode_3_2025-01-06-17-10-36.bag filter=lfs diff=lfs merge=lfs -text +3d3i07nt/episode_rosbags/episode_0_2025-01-06-21-01-14.bag filter=lfs diff=lfs merge=lfs -text +hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag b/3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..0c33e012000431b9a9745986bf24557406eae55b --- /dev/null +++ b/3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d0e30340688ce376a6e1d057c860321442860c55868a23e5497c207a7d9a69f +size 1411056389 diff --git a/3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag b/3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..741a21fb3ad6010d6e5227f44dd04e76a41002b9 --- /dev/null +++ b/3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:163177de8dd4d24d90cba9bb06b389f6f2e4d3329446c0e0d8164eb2013358eb +size 1412673424 diff --git a/3d3i07nt/.hydra/config.yaml b/3d3i07nt/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/3d3i07nt/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/3d3i07nt/.hydra/hydra.yaml b/3d3i07nt/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b4886357cd92499edd4d506bfd172f007e156f1 --- /dev/null +++ b/3d3i07nt/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/3d3i07nt/.hydra/overrides.yaml b/3d3i07nt/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/3d3i07nt/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/3d3i07nt/episode_rosbags/aligned_depth_to_color_K.npy b/3d3i07nt/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3d3i07nt/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3d3i07nt/episode_rosbags/cam_tf_world.npy b/3d3i07nt/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/3d3i07nt/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/3d3i07nt/episode_rosbags/color_K.npy b/3d3i07nt/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/3d3i07nt/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/3d3i07nt/episode_rosbags/depth_K.npy b/3d3i07nt/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/3d3i07nt/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/3d3i07nt/episode_rosbags/episode_0_2025-01-06-21-01-14.bag b/3d3i07nt/episode_rosbags/episode_0_2025-01-06-21-01-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..6c409c142fc1f7b3bfa89006f6f8fbf6043b274a --- /dev/null +++ b/3d3i07nt/episode_rosbags/episode_0_2025-01-06-21-01-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e0fd87ee9b598880245ea478ac235b88d3c4960676ff4dce3fe1258b14b7de +size 1406802429 diff --git a/3d3i07nt/episode_rosbags/episode_1_2025-01-06-21-01-55.bag b/3d3i07nt/episode_rosbags/episode_1_2025-01-06-21-01-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..83ebd0602422776fcf253c6591043f08cad62c6e --- /dev/null +++ b/3d3i07nt/episode_rosbags/episode_1_2025-01-06-21-01-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:619f47f2b632c7f6d219297fb28b16cc57b759c75e320dad1555879714886b4d +size 1412368817 diff --git a/3d3i07nt/episode_rosbags/episode_3_2025-01-06-21-03-44.bag b/3d3i07nt/episode_rosbags/episode_3_2025-01-06-21-03-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..8014dd0830747dd9a7635e0012d768150041d1b6 --- /dev/null +++ b/3d3i07nt/episode_rosbags/episode_3_2025-01-06-21-03-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:679727261d8de65c9e5110ae38d09bece3b605e8486d4ec142a37882fa27da73 +size 1409876037 diff --git a/3d3i07nt/episode_rosbags/episode_4_2025-01-06-21-04-27.bag b/3d3i07nt/episode_rosbags/episode_4_2025-01-06-21-04-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..a69c62ab593c04388ca135e55d3c43450625782c --- /dev/null +++ b/3d3i07nt/episode_rosbags/episode_4_2025-01-06-21-04-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1164f9fe52f1fd8e5eed63689d35896b4739da6d7f7da4b9576d519f2387058 +size 1411827048 diff --git a/3d3i07nt/eval_robot.log b/3d3i07nt/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..8cc485739cbde259015b822e34f75b90511a81f1 --- /dev/null +++ b/3d3i07nt/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 21:00:46,850][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 21:00:46,853][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/3d3i07nt/eval_video/0_eval.mp4 b/3d3i07nt/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c8653835f53323cef51abac71a54e6ddc70b33d1 --- /dev/null +++ b/3d3i07nt/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bab73a244800e0004931ca0f6cf06f7b7ab3cda2c120d4f9b0ebb2346c0e61b +size 1129917 diff --git a/3d3i07nt/eval_video/1_eval.mp4 b/3d3i07nt/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8344484372e94d6d8e6263b8563f2f6243cd4cd6 --- /dev/null +++ b/3d3i07nt/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cbaf6111e2b85cffc2c086f5058e8c60408cfd924477d545569a93cee30fb07 +size 1190108 diff --git a/3d3i07nt/eval_video/2_eval.mp4 b/3d3i07nt/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68ae11c87752e62d280fd1cacaea3f0c81c6eaff --- /dev/null +++ b/3d3i07nt/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:559e440a12c1c69e9e1a6abfcaa2f93fd1c605f9e31554a764a92674f93d7d91 +size 1199446 diff --git a/3d3i07nt/eval_video/3_eval.mp4 b/3d3i07nt/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e807c2aa4b97eeae33569d1adcffefb4cff6ec2 --- /dev/null +++ b/3d3i07nt/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a07183c07b2b4777782e87161a734c79d9608576ae4002c683a78f0c9fac2b4 +size 1138695 diff --git a/3d3i07nt/eval_video/4_eval.mp4 b/3d3i07nt/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..caa9e9cc28c36de4a5a2ccb0fd397554c8003e63 --- /dev/null +++ b/3d3i07nt/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51249e60c287ad366a8fa693d1684fa97bc8a0f60d9bf571d2be5ba7a2d07aa +size 1254034 diff --git a/3d3i07nt/tb/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 b/3d3i07nt/tb/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 new file mode 100644 index 0000000000000000000000000000000000000000..f4d656bae1369fb8f5b51f72a377fe274dcee612 --- /dev/null +++ b/3d3i07nt/tb/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe4fb38b4e629f3194ab701b0deabc21d37214eee994715f7742c5e7f977cbc +size 6539 diff --git a/3d3i07nt/wandb/debug-internal.log b/3d3i07nt/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..03bba301fe3fbf4a2ddeab21dfd1204313706a41 --- /dev/null +++ b/3d3i07nt/wandb/debug-internal.log @@ -0,0 +1,779 @@ +2025-01-06 21:00:53,097 INFO StreamThr :2117937 [internal.py:wandb_internal():85] W&B internal server running at pid: 2117937, started at: 2025-01-06 21:00:53.097352 +2025-01-06 21:00:53,099 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status +2025-01-06 21:00:53,111 INFO WriterThread:2117937 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb +2025-01-06 21:00:53,112 DEBUG SenderThread:2117937 [sender.py:send():379] send: header +2025-01-06 21:00:53,120 DEBUG SenderThread:2117937 [sender.py:send():379] send: run +2025-01-06 21:00:53,382 INFO SenderThread:2117937 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files +2025-01-06 21:00:53,382 INFO SenderThread:2117937 [sender.py:_start_run_threads():1188] run started: 3d3i07nt with start time 1736215253.098176 +2025-01-06 21:00:53,401 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 21:00:53,401 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: check_version +2025-01-06 21:00:53,489 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 21:00:53,494 DEBUG HandlerThread:2117937 [system_info.py:__init__():26] System info init +2025-01-06 21:00:53,494 DEBUG HandlerThread:2117937 [system_info.py:__init__():41] System info init done +2025-01-06 21:00:53,494 INFO HandlerThread:2117937 [system_monitor.py:start():194] Starting system monitor +2025-01-06 21:00:53,494 INFO SystemMonitor:2117937 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 21:00:53,494 INFO HandlerThread:2117937 [system_monitor.py:probe():214] Collecting system info +2025-01-06 21:00:53,495 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 21:00:53,495 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started disk monitoring +2025-01-06 21:00:53,496 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 21:00:53,496 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started memory monitoring +2025-01-06 21:00:53,497 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started network monitoring +2025-01-06 21:00:53,507 DEBUG HandlerThread:2117937 [system_info.py:probe():152] Probing system +2025-01-06 21:00:53,509 DEBUG HandlerThread:2117937 [system_info.py:_probe_git():137] Probing git +2025-01-06 21:00:53,513 DEBUG HandlerThread:2117937 [system_info.py:_probe_git():145] Probing git done +2025-01-06 21:00:53,513 DEBUG HandlerThread:2117937 [system_info.py:probe():200] Probing system done +2025-01-06 21:00:53,514 DEBUG HandlerThread:2117937 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T02:00:53.507771', 'startedAt': '2025-01-07T02:00:53.088016', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.778875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 538.4776840209961}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 21:00:53,514 INFO HandlerThread:2117937 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 21:00:53,514 INFO HandlerThread:2117937 [system_monitor.py:probe():227] Publishing system info +2025-01-06 21:00:53,514 DEBUG HandlerThread:2117937 [system_info.py:_save_code():44] Saving code +2025-01-06 21:00:53,515 DEBUG HandlerThread:2117937 [system_info.py:_save_code():67] Saving code done +2025-01-06 21:00:53,515 DEBUG HandlerThread:2117937 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 21:00:53,584 DEBUG HandlerThread:2117937 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 21:00:53,584 INFO HandlerThread:2117937 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 21:00:53,587 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:00:53,587 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 21:00:53,587 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 21:00:53,588 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 21:00:53,840 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/cwxs2uff-code/FISH/eval_robot.py +2025-01-06 21:00:53,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 21:00:53,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: python_packages +2025-01-06 21:00:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:53,853 DEBUG SenderThread:2117937 [sender.py:send():379] send: telemetry +2025-01-06 21:00:53,853 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/e5jwekfy-wandb-metadata.json +2025-01-06 21:00:53,875 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/7qu68wlj-diff.patch +2025-01-06 21:00:53,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:00:53,893 INFO HandlerThread:2117937 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/tb" + save: true +} +control { + relay_id: "ced5b79432f94c96ac79791dbb41a3be" +} +_info { + stream_id: "3d3i07nt" +} + +2025-01-06 21:00:53,894 DEBUG HandlerThread:2117937 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 21:00:53,894 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:00:54,013 DEBUG SenderThread:2117937 [sender.py:send():379] send: tbrecord +2025-01-06 21:00:54,014 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:00:54,014 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 with policy live +2025-01-06 21:00:54,205 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/dijz3mrs-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code +2025-01-06 21:00:54,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:55,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:56,383 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:00:56,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:57,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:58,766 DEBUG SenderThread:2117937 [sender.py:send():379] send: config +2025-01-06 21:00:58,766 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:00:58,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:59,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:00,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:00,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:01,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:02,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:02,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:03,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:03,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:04,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:04,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:05,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:06,386 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:06,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:07,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:08,387 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:08,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:08,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:08,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:08,992 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:09,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:10,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:11,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:12,388 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:12,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:13,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:14,389 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:14,579 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:14,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:15,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:16,389 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:16,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:17,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:18,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:19,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:19,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:20,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:21,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:22,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:23,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:23,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:23,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:24,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:25,015 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:25,392 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:01:25,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:26,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:27,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:28,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:29,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:30,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:30,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:31,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:32,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:33,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:34,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:35,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:36,395 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:36,685 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:36,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:37,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:38,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:38,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:38,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:39,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:40,396 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:40,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:41,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:42,319 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:42,530 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:01:42,531 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 with policy now +2025-01-06 21:01:42,531 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:01:42,532 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:01:42,532 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:01:42,537 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:01:42,800 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/inojqhlt-media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:42,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:43,397 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:01:43,686 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/xd62vwum-media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:43,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:44,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:44,859 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:45,859 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:46,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:47,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:48,185 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:48,399 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:48,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:49,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:50,400 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:50,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:51,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:52,401 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:01:52,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:53,497 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 21:01:53,498 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:01:53,498 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:53,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:53,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:54,401 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:54,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:55,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:56,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:57,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:58,403 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:58,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:58,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:59,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:00,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:01,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:02,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:03,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:03,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:04,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:05,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:06,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:07,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:08,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:08,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:08,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:08,918 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:09,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:10,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:11,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:12,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:13,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:14,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:14,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:15,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:16,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:17,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:18,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:19,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:19,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:20,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:21,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:22,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:23,507 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:02:23,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:23,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:23,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:24,179 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:02:24,179 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 with policy now +2025-01-06 21:02:24,182 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:02:24,183 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:02:24,183 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:02:24,184 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:02:24,448 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/dnchds9h-media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,720 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/fbs9frno-media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,868 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:25,201 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:25,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:26,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:26,868 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:27,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:28,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:29,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:30,413 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:30,448 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:30,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:31,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:32,413 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:02:32,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:32,675 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/zupl7yxz-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:02:32,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:33,372 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:02:33,372 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:02:33,373 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:02:33,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:02:33,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:34,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:34,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:35,825 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:35,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:36,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:37,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:38,415 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:38,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:38,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:38,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:39,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:40,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:41,009 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:41,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:42,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:43,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:44,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:45,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:46,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:46,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:47,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:48,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:49,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:50,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:51,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:51,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:52,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:53,499 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:02:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:53,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:53,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:54,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:55,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:56,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:56,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:57,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:58,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:59,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:00,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:01,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:02,778 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:02,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:03,882 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:04,249 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:03:04,249 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 with policy now +2025-01-06 21:03:04,251 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:03:04,252 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:03:04,252 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:03:04,252 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:03:04,503 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/nnw7v8qe-media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,796 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/cr1k5tmp-media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,879 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:05,884 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:06,425 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:06,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:07,889 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:08,425 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:08,621 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:08,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:08,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:08,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:09,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:10,884 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:11,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:12,427 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:12,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:13,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:13,885 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:14,886 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:15,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:16,891 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:17,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:18,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:18,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:19,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:20,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:21,891 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:22,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:23,499 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:03:23,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:23,856 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:23,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:23,973 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:24,899 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:25,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:26,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:27,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:28,890 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:29,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:29,892 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:30,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:31,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:32,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:33,895 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:34,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:35,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:35,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:36,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:37,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:38,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:38,861 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:38,909 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:39,899 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:40,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:41,147 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:03:41,147 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:03:41,148 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:41,148 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:03:41,436 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:03:41,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:03:41,680 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/uj9tdk34-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:03:41,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:42,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:42,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:43,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:44,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:44,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:45,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:46,438 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:46,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:47,004 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:47,902 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:48,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:49,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:50,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:51,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:52,862 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:52,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:53,500 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:03:53,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:53,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:53,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:54,441 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:54,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:55,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:56,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:57,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:57,995 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:58,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:59,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:00,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:01,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:02,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:03,862 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:03,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:04,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:05,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:06,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:07,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:08,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:08,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:08,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:08,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:09,909 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:10,446 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:10,910 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:11,913 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:12,447 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:12,917 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:13,230 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:04:13,230 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 with policy now +2025-01-06 21:04:13,243 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:04:13,243 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:13,243 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:13,244 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:04:13,492 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/8e25x4z2-media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,760 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/wy76zwe0-media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,923 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:14,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:14,866 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:14,917 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:15,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:16,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:17,924 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:18,449 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:18,922 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:19,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:20,409 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:20,450 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:20,928 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:21,927 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:22,924 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:23,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:04:23,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:23,860 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:23,922 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:24,200 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:24,200 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:24,201 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:24,451 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:04:24,452 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:24,930 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:25,929 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:26,117 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:26,452 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:26,938 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:27,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:28,940 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:29,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:30,454 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:30,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:31,591 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:31,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:32,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:33,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:34,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:35,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:36,867 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:36,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:37,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:38,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:38,857 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:38,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:39,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:40,457 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:40,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:41,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:41,978 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:42,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:43,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:44,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:45,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:46,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:47,867 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:47,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:48,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:49,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:50,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:51,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:52,870 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:52,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:53,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:04:53,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:53,857 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:53,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:54,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:55,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:56,939 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:57,669 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:04:57,669 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 with policy now +2025-01-06 21:04:57,674 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:04:57,674 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:57,674 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:57,675 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:57,937 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/w0jcvthq-media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:57,938 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:58,479 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:04:58,706 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:58,770 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/05y7ayc2-media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:58,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:59,943 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:00,949 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:01,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:02,479 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:02,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:03,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:03,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:04,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:05,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:06,481 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:06,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:07,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:08,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:05:08,864 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:05:08,951 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:08,989 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:09,957 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:10,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:11,959 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:12,951 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:13,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:14,878 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:14,959 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:15,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:16,955 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:17,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:18,970 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:19,882 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:19,965 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:20,968 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:21,965 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:22,969 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:23,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:05:23,870 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:05:23,871 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:05:23,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:23,998 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:05:24,011 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:05:24,268 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:05:24,474 INFO wandb-upload_2:2117937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7ygkp7hl +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media +2025-01-06 21:05:24,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval +2025-01-06 21:05:24,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table +2025-01-06 21:05:24,679 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:05:24,687 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:05:24,959 INFO SenderThread:2117937 [sender.py:send_request_log_artifact():1525] logged artifact run-3d3i07nt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM3MDQwNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2NTMxOQ==', 'latestArtifact': None}} +2025-01-06 21:05:24,959 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:05:24,961 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:24,961 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-06 21:05:24,961 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:24,961 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,962 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,962 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:24,962 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:24,963 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send():379] send: telemetry +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,963 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,964 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,964 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,965 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,965 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,965 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,965 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,966 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,966 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,967 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,967 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,967 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,967 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,968 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,968 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:05:25,205 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/nesakpao-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:25,210 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/fadlkzzd-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:25,369 INFO wandb-upload_2:2117937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwont73sm +2025-01-06 21:05:25,449 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/jmvf3due-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:25,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval +2025-01-06 21:05:25,733 INFO SenderThread:2117937 [sender.py:send_request_log_artifact():1525] logged artifact run-3d3i07nt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM3MDU1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2NTQwMQ==', 'latestArtifact': None}} +2025-01-06 21:05:25,733 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:05:25,733 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json with policy now +2025-01-06 21:05:25,733 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:25,734 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:25,734 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:25,908 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/g94hhcqb-media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:25,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:26,157 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/h17ehwwa-media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:26,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:26,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:26,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:27,963 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:28,963 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:29,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:30,005 DEBUG SenderThread:2117937 [sender.py:send():379] send: exit +2025-01-06 21:05:30,005 INFO SenderThread:2117937 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_exit():588] handling runtime: 276 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_exit():594] send defer +2025-01-06 21:05:30,006 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,006 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 21:05:30,006 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 1 +2025-01-06 21:05:30,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 21:05:30,007 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,007 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 21:05:30,007 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 2 +2025-01-06 21:05:30,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 21:05:30,007 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 21:05:30,008 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 21:05:30,008 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined network monitor +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 3 +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:05:30,060 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,060 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 4 +2025-01-06 21:05:30,060 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,061 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 21:05:30,489 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:32,490 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:36,277 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:36,278 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,278 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 21:05:36,278 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 5 +2025-01-06 21:05:36,278 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:36,278 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:36,278 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 21:05:36,279 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,279 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 21:05:36,280 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:36,280 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:36,281 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:36,281 DEBUG SenderThread:2117937 [sender.py:send():379] send: summary +2025-01-06 21:05:36,285 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:36,285 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,285 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 21:05:36,286 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 6 +2025-01-06 21:05:36,286 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,286 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 21:05:36,286 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,286 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 21:05:36,289 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:36,480 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 7 +2025-01-06 21:05:36,481 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,481 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 21:05:36,481 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,481 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 21:05:36,492 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:05:36,492 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:37,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:40,281 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 8 +2025-01-06 21:05:40,281 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:40,281 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,282 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 21:05:40,282 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,282 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 21:05:40,282 INFO SenderThread:2117937 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 21:05:40,284 INFO SenderThread:2117937 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 21:05:40,286 INFO SenderThread:2117937 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 21:05:40,300 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 9 +2025-01-06 21:05:40,300 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,301 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 21:05:40,301 DEBUG SenderThread:2117937 [sender.py:send():379] send: artifact +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 21:05:40,489 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 21:05:40,493 INFO SenderThread:2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:40,493 INFO SenderThread:2117937 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml config.yaml +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json wandb-summary.json +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt requirements.txt +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json wandb-metadata.json +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log output.log +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch diff.patch +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 10 +2025-01-06 21:05:40,499 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,503 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 21:05:40,504 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,505 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 21:05:40,505 INFO SenderThread:2117937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:05:40,675 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:05:40,707 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:40,728 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/mnxincra-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:40,731 INFO wandb-upload_4:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:40,733 INFO wandb-upload_3:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt +2025-01-06 21:05:40,934 INFO Thread-11 (_thread_body):2117937 [sender.py:transition_state():614] send defer: 11 +2025-01-06 21:05:40,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,934 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 21:05:40,934 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,935 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 21:05:40,935 INFO SenderThread:2117937 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 12 +2025-01-06 21:05:40,936 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,936 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 21:05:40,936 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [file_stream.py:finish():601] file stream finish called +2025-01-06 21:05:41,008 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [file_stream.py:finish():605] file stream finish is done +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 13 +2025-01-06 21:05:41,162 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,162 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:41,162 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 21:05:41,162 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 14 +2025-01-06 21:05:41,162 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:41,162 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send():379] send: final +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send():379] send: footer +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:41,163 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 21:05:41,163 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,164 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 21:05:41,164 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,164 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: server_info +2025-01-06 21:05:41,166 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,166 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 21:05:41,167 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 21:05:41,167 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:41,229 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 21:05:41,231 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 21:05:41,231 INFO HandlerThread:2117937 [handler.py:finish():882] shutting down handler +2025-01-06 21:05:42,167 INFO WriterThread:2117937 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [sender.py:finish():1615] shutting down sender +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [file_pusher.py:join():175] waiting for file pusher diff --git a/3d3i07nt/wandb/debug.log b/3d3i07nt/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d748241179217175b8a4f06bb6fc75fb673fe319 --- /dev/null +++ b/3d3i07nt/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Configure stats pid to 2117773 +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/settings +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug.log +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug-internal.log +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():569] calling init triggers +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt'} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():619] starting backend +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():623] setting up manager +2025-01-06 21:00:53,096 INFO MainThread:2117773 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 21:00:53,097 INFO MainThread:2117773 [wandb_init.py:init():631] backend started and connected +2025-01-06 21:00:53,111 INFO MainThread:2117773 [wandb_init.py:init():720] updated telemetry +2025-01-06 21:00:53,117 INFO MainThread:2117773 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 21:00:53,401 INFO MainThread:2117773 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 21:00:53,441 INFO MainThread:2117773 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 21:00:53,441 INFO MainThread:2117773 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 21:00:53,853 INFO MainThread:2117773 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 21:00:53,854 INFO MainThread:2117773 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/tb, True +2025-01-06 21:00:58,766 INFO MainThread:2117773 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 21:05:42,231 WARNING MsgRouterThr:2117773 [router.py:message_loop():77] message_loop has been closed diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af5dbe2ff5cbf1dc5c5b50fd0b0a91eccd82a8fa --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736215253 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]} \ No newline at end of file diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json new file mode 100644 index 0000000000000000000000000000000000000000..c15d43ce456f45242f5b08699c075e78557fee53 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[19, 30, 0.6333333333333333, 0.41478991510188096, 0.8161348401867883, 0.4590426236472983, 0.7995879947540014, 0.48694152385257594, 0.7620869948935856, 0.5056970939028789, 0.7494037709631406]]} \ No newline at end of file diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c8653835f53323cef51abac71a54e6ddc70b33d1 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bab73a244800e0004931ca0f6cf06f7b7ab3cda2c120d4f9b0ebb2346c0e61b +size 1129917 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8344484372e94d6d8e6263b8563f2f6243cd4cd6 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cbaf6111e2b85cffc2c086f5058e8c60408cfd924477d545569a93cee30fb07 +size 1190108 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68ae11c87752e62d280fd1cacaea3f0c81c6eaff --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:559e440a12c1c69e9e1a6abfcaa2f93fd1c605f9e31554a764a92674f93d7d91 +size 1199446 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6e807c2aa4b97eeae33569d1adcffefb4cff6ec2 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a07183c07b2b4777782e87161a734c79d9608576ae4002c683a78f0c9fac2b4 +size 1138695 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..caa9e9cc28c36de4a5a2ccb0fd397554c8003e63 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51249e60c287ad366a8fa693d1684fa97bc8a0f60d9bf571d2be5ba7a2d07aa +size 1254034 diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..29fc07454aeb24942d413d5462071e33c39918e1 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log @@ -0,0 +1,288 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736215258.766597]: resetting environment +[INFO] [1736215258.771973]: cleared current plan +[INFO] [1736215258.773120]: moving to home +[INFO] [1736215259.775630]: reached home +[INFO] [1736215259.776827]: reset action history +[INFO] [1736215262.484371]: environment reset +Starting episode 0 +[INFO] [1736215262.486110]: resetting environment +[INFO] [1736215262.488724]: cleared current plan +[INFO] [1736215262.489758]: moving to home +[INFO] [1736215263.490935]: reached home +[INFO] [1736215263.492842]: reset action history +[INFO] [1736215266.202095]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736215270.862028]: resetting environment +[INFO] [1736215270.865521]: cleared current plan +[INFO] [1736215270.866416]: moving to home +[INFO] [1736215271.868576]: reached home +[INFO] [1736215271.869833]: reset action history +[INFO] [1736215274.579387]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/episode_rosbags/episode_0_2025-01-06-21-01-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2117773.log +started roslaunch server http://158.130.50.37:35077/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2118037] +process[print_text-2]: started with pid [2118061] +process[rosbag_record-3]: started with pid [2118062] +started bagging! +For topic gripper_width: timestamp difference is -112819641 for nearest: 1736215294433182985 - target: 1736215294546002626 at idx 249 +For topic gripper_width: timestamp difference is -104516037 for nearest: 1736215298433264725 - target: 1736215298537780762 at idx 249 +For topic gripper_width: timestamp difference is -102902324 for nearest: 1736215299233216136 - target: 1736215299336118460 at idx 249 +[INFO] [1736215301.592463]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1736215302.318407]: Stored episode 1. +[INFO] [1736215302.319490]: Saving video... +[INFO] [1736215302.518716]: Video saved! +Starting episode 1 +[INFO] [1736215302.532152]: resetting environment +[INFO] [1736215302.537164]: cleared current plan +[INFO] [1736215302.538987]: moving to home +[INFO] [1736215306.141250]: reached home +[INFO] [1736215306.141756]: reset action history +[INFO] [1736215308.897845]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215312.017663]: resetting environment +[INFO] [1736215312.024826]: cleared current plan +[INFO] [1736215312.025013]: moving to home +[INFO] [1736215313.026850]: reached home +[INFO] [1736215313.027822]: reset action history +[INFO] [1736215315.734277]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/episode_rosbags/episode_1_2025-01-06-21-01-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2117773.log +started roslaunch server http://158.130.50.37:40811/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2118170] +process[print_text-5]: started with pid [2118194] +process[rosbag_record-6]: started with pid [2118195] +started bagging! +[INFO] [1736215342.728932]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736215343.502403]: Stored episode 2. +[INFO] [1736215343.502620]: Saving video... +[WARN] [1736215344.103765]: Plan exhausted +[WARN] [1736215344.144446]: Plan exhausted +[WARN] [1736215344.185148]: Plan exhausted +[INFO] [1736215344.154313]: Video saved! +Starting episode 2 +[INFO] [1736215344.186952]: resetting environment +[INFO] [1736215344.196701]: cleared current plan +[INFO] [1736215344.201097]: moving to home +[INFO] [1736215347.709324]: reached home +[INFO] [1736215347.714700]: reset action history +[INFO] [1736215350.447148]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215352.103048]: resetting environment +[INFO] [1736215352.113539]: cleared current plan +[INFO] [1736215352.114653]: moving to home +[INFO] [1736215353.116683]: reached home +[INFO] [1736215353.117227]: reset action history +[INFO] [1736215355.824891]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/episode_rosbags/episode_2_2025-01-06-21-02-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2117773.log +started roslaunch server http://158.130.50.37:41479/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2118310] +process[print_text-8]: started with pid [2118311] +process[rosbag_record-9]: started with pid [2118312] +started bagging! +[INFO] [1736215382.778011]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736215383.564737]: Stored episode 3. +[INFO] [1736215383.569938]: Saving video... +[WARN] [1736215383.903778]: Plan exhausted +[WARN] [1736215383.943565]: Plan exhausted +[WARN] [1736215383.985212]: Plan exhausted +[WARN] [1736215384.023910]: Plan exhausted +[WARN] [1736215384.064683]: Plan exhausted +[WARN] [1736215384.103684]: Plan exhausted +[WARN] [1736215384.152051]: Plan exhausted +[WARN] [1736215384.183723]: Plan exhausted +[WARN] [1736215384.224017]: Plan exhausted +[WARN] [1736215384.263572]: Plan exhausted +[INFO] [1736215384.210249]: Video saved! +Starting episode 3 +[INFO] [1736215384.254754]: resetting environment +[INFO] [1736215384.267194]: cleared current plan +[INFO] [1736215384.267423]: moving to home +[INFO] [1736215387.573491]: reached home +[INFO] [1736215387.576970]: reset action history +[INFO] [1736215390.361037]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215420.832452]: resetting environment +[INFO] [1736215420.847285]: cleared current plan +[INFO] [1736215420.849867]: moving to home +[INFO] [1736215421.858603]: reached home +[INFO] [1736215421.862270]: reset action history +[INFO] [1736215424.747177]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/episode_rosbags/episode_3_2025-01-06-21-03-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2117773.log +started roslaunch server http://158.130.50.37:33489/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2118456] +process[print_text-11]: started with pid [2118457] +process[rosbag_record-12]: started with pid [2118481] +started bagging! +For topic gripper_width: timestamp difference is 104037188 for nearest: 1736215433333215855 - target: 1736215433229178667 at idx 118 +For topic gripper_width: timestamp difference is 116264490 for nearest: 1736215448533219007 - target: 1736215448416954517 at idx 249 +For topic gripper_width: timestamp difference is 118684011 for nearest: 1736215451533173042 - target: 1736215451414489031 at idx 249 +[INFO] [1736215451.716600]: Storing episode... +[WARN] [1736215453.032855]: Plan exhausted +[WARN] [1736215453.063924]: Plan exhausted +[WARN] [1736215453.103744]: Plan exhausted +[WARN] [1736215453.149259]: Plan exhausted +[WARN] [1736215453.185850]: Plan exhausted +[WARN] [1736215453.228110]: Plan exhausted +[WARN] [1736215453.264341]: Plan exhausted +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736215452.508287]: Stored episode 4. +[INFO] [1736215452.511482]: Saving video... +[INFO] [1736215453.190585]: Video saved! +Starting episode 4 +[INFO] [1736215453.248066]: resetting environment +[INFO] [1736215453.271276]: cleared current plan +[INFO] [1736215453.272460]: moving to home +[INFO] [1736215456.580319]: reached home +[INFO] [1736215456.585137]: reset action history +[INFO] [1736215459.362964]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215464.061521]: resetting environment +[INFO] [1736215464.067383]: cleared current plan +[INFO] [1736215464.067554]: moving to home +[INFO] [1736215465.069118]: reached home +[INFO] [1736215465.076427]: reset action history +[INFO] [1736215467.859360]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/episode_rosbags/episode_4_2025-01-06-21-04-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2117773.log +started roslaunch server http://158.130.50.37:37291/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2118616] +process[print_text-14]: started with pid [2118617] +process[rosbag_record-15]: started with pid [2118618] +started bagging! +For topic gripper_width: timestamp difference is -102234722 for nearest: 1736215478133240580 - target: 1736215478235475302 at idx 167 +[WARN] [1736215497.665156]: Plan exhausted +[INFO] [1736215496.298033]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1736215497.071206]: Stored episode 5. +[INFO] [1736215497.076669]: Saving video... +[INFO] [1736215497.632148]: Video saved! +[INFO] [1736215497.674372]: resetting environment +[INFO] [1736215497.701562]: cleared current plan +[INFO] [1736215497.703598]: moving to home +[INFO] [1736215501.311147]: reached home +[INFO] [1736215501.317502]: reset action history +[INFO] [1736215504.098241]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..781e2a26c24979d02219b57c9d512e62e2fcef18 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T02:00:53.507771", + "startedAt": "2025-01-07T02:00:53.088016", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.778875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.019, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 538.4776840209961 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b5b96ecebd2c5e975d4554a91741e13e020faf57 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3bab73a244800e0004931ca0f6cf06f7b7ab3cda2c120d4f9b0ebb2346c0e61b", "size": 1129917, "path": "media/videos/eval/0_eval_0_3bab73a244800e000493.mp4"}, "global_step": 664, "_timestamp": 1736215524.588005, "_runtime": 271.4898290634155, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9cbaf6111e2b85cffc2c086f5058e8c60408cfd924477d545569a93cee30fb07", "size": 1190108, "path": "media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4"}, "eval/num_success": 3.0, "episode": 30.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "559e440a12c1c69e9e1a6abfcaa2f93fd1c605f9e31554a764a92674f93d7d91", "size": 1199446, "path": "media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9a07183c07b2b4777782e87161a734c79d9608576ae4002c683a78f0c9fac2b4", "size": 1138695, "path": "media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d51249e60c287ad366a8fa693d1684fa97bc8a0f60d9bf571d2be5ba7a2d07aa", "size": 1254034, "path": "media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://hbyqhmskn3nwtovymb5ww1l2fav4rump0tjfk22bqk4fzlyz4kyfa64vhytltm4jwzpdme755f5w4igwtnmn76hm6b8g463aa7hdkgyadrq1e1qet252bymrnnicm74n/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://j7djuiee7xum3rkd2soqehqy1b69qjnfbtl8sd3q3pquq4i5xpdnkj83wzqpvd7sq2eqm7noqe3mku2pd93h8wptiqe84rovfwlmhjq7hslgdsck8wa9o29g7o5ukt4v:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 19, "total_num_episodes": 30, "total_success_rate": 0.6333333333333333, "total_success_rate_lb_0.01": 0.41478991510188096, "total_success_rate_ub_0.01": 0.8161348401867883, "total_success_rate_lb_0.025": 0.4590426236472983, "total_success_rate_ub_0.025": 0.7995879947540014, "total_success_rate_lb_0.05": 0.48694152385257594, "total_success_rate_ub_0.05": 0.7620869948935856, "total_success_rate_lb_0.1": 0.5056970939028789, "total_success_rate_ub_0.1": 0.7494037709631406, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "b2a679debf83a2b3c3cf7186730f541eb3930df19cf2cb6b196d309604a208fc", "size": 522, "artifact_path": "wandb-client-artifact://uwi6178e3y0yxremi8hieghrrl172uos2qg6k6k8wvc1ijner9tajcpvvgfcsxe63v0b2r3jbh112u44i5j5s7ijgudzsaque1dz93bvdnmndl7ebfjiukb42jtkz8lz/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ei90bjbw21zqk40z40gy3vfzvt0jwzqurpa2dnrgikciv6tulfhhe94iagor2eg5ack1c5ub1fxgdzba6074t1gciy1e13dv0p957e38lkubfp08ha60sm8gngvm1zod:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 276}, "eval/total_success_rate": 0.6333333253860474, "eval/total_success_rate_lb_0.01": 0.41478991508483887, "eval/total_success_rate_ub_0.01": 0.8161348104476929, "eval/total_success_rate_lb_0.025": 0.45904263854026794, "eval/total_success_rate_ub_0.025": 0.7995879650115967, "eval/total_success_rate_lb_0.05": 0.48694151639938354, "eval/total_success_rate_ub_0.05": 0.7620869874954224, "eval/total_success_rate_lb_0.1": 0.5056970715522766, "eval/total_success_rate_ub_0.1": 0.7494037747383118} \ No newline at end of file diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug-internal.log b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..03bba301fe3fbf4a2ddeab21dfd1204313706a41 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug-internal.log @@ -0,0 +1,779 @@ +2025-01-06 21:00:53,097 INFO StreamThr :2117937 [internal.py:wandb_internal():85] W&B internal server running at pid: 2117937, started at: 2025-01-06 21:00:53.097352 +2025-01-06 21:00:53,099 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status +2025-01-06 21:00:53,111 INFO WriterThread:2117937 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb +2025-01-06 21:00:53,112 DEBUG SenderThread:2117937 [sender.py:send():379] send: header +2025-01-06 21:00:53,120 DEBUG SenderThread:2117937 [sender.py:send():379] send: run +2025-01-06 21:00:53,382 INFO SenderThread:2117937 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files +2025-01-06 21:00:53,382 INFO SenderThread:2117937 [sender.py:_start_run_threads():1188] run started: 3d3i07nt with start time 1736215253.098176 +2025-01-06 21:00:53,401 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 21:00:53,401 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: check_version +2025-01-06 21:00:53,489 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 21:00:53,494 DEBUG HandlerThread:2117937 [system_info.py:__init__():26] System info init +2025-01-06 21:00:53,494 DEBUG HandlerThread:2117937 [system_info.py:__init__():41] System info init done +2025-01-06 21:00:53,494 INFO HandlerThread:2117937 [system_monitor.py:start():194] Starting system monitor +2025-01-06 21:00:53,494 INFO SystemMonitor:2117937 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 21:00:53,494 INFO HandlerThread:2117937 [system_monitor.py:probe():214] Collecting system info +2025-01-06 21:00:53,495 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 21:00:53,495 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started disk monitoring +2025-01-06 21:00:53,496 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 21:00:53,496 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started memory monitoring +2025-01-06 21:00:53,497 INFO SystemMonitor:2117937 [interfaces.py:start():188] Started network monitoring +2025-01-06 21:00:53,507 DEBUG HandlerThread:2117937 [system_info.py:probe():152] Probing system +2025-01-06 21:00:53,509 DEBUG HandlerThread:2117937 [system_info.py:_probe_git():137] Probing git +2025-01-06 21:00:53,513 DEBUG HandlerThread:2117937 [system_info.py:_probe_git():145] Probing git done +2025-01-06 21:00:53,513 DEBUG HandlerThread:2117937 [system_info.py:probe():200] Probing system done +2025-01-06 21:00:53,514 DEBUG HandlerThread:2117937 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T02:00:53.507771', 'startedAt': '2025-01-07T02:00:53.088016', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.778875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 538.4776840209961}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 21:00:53,514 INFO HandlerThread:2117937 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 21:00:53,514 INFO HandlerThread:2117937 [system_monitor.py:probe():227] Publishing system info +2025-01-06 21:00:53,514 DEBUG HandlerThread:2117937 [system_info.py:_save_code():44] Saving code +2025-01-06 21:00:53,515 DEBUG HandlerThread:2117937 [system_info.py:_save_code():67] Saving code done +2025-01-06 21:00:53,515 DEBUG HandlerThread:2117937 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 21:00:53,584 DEBUG HandlerThread:2117937 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 21:00:53,584 INFO HandlerThread:2117937 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 21:00:53,587 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:00:53,587 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 21:00:53,587 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 21:00:53,588 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 21:00:53,840 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/cwxs2uff-code/FISH/eval_robot.py +2025-01-06 21:00:53,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 21:00:53,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: python_packages +2025-01-06 21:00:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:53,853 DEBUG SenderThread:2117937 [sender.py:send():379] send: telemetry +2025-01-06 21:00:53,853 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/e5jwekfy-wandb-metadata.json +2025-01-06 21:00:53,875 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/7qu68wlj-diff.patch +2025-01-06 21:00:53,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:00:53,893 INFO HandlerThread:2117937 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/tb" + save: true +} +control { + relay_id: "ced5b79432f94c96ac79791dbb41a3be" +} +_info { + stream_id: "3d3i07nt" +} + +2025-01-06 21:00:53,894 DEBUG HandlerThread:2117937 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 21:00:53,894 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:00:54,013 DEBUG SenderThread:2117937 [sender.py:send():379] send: tbrecord +2025-01-06 21:00:54,014 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:00:54,014 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 with policy live +2025-01-06 21:00:54,205 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/dijz3mrs-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:00:54,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH +2025-01-06 21:00:54,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code +2025-01-06 21:00:54,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:55,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:56,383 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:00:56,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:57,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:58,766 DEBUG SenderThread:2117937 [sender.py:send():379] send: config +2025-01-06 21:00:58,766 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:00:58,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:00:59,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:00,384 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:00,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:01,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:02,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:02,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:03,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:03,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:04,385 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:04,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:05,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:06,386 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:06,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:07,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:08,387 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:08,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:08,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:08,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:08,992 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:09,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:10,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:11,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:12,388 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:12,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:13,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:14,389 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:14,579 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:14,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:15,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:16,389 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:16,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:17,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:18,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:19,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:19,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:20,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:21,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:22,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:23,851 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:23,851 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:23,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:24,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:25,015 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:25,392 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:01:25,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:26,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:27,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:28,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:29,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:30,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:30,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:31,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:32,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:33,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:34,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:35,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:36,395 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:36,685 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:36,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:37,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:38,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:38,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:38,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:39,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:40,396 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:40,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:41,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:42,319 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:42,530 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:01:42,531 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 with policy now +2025-01-06 21:01:42,531 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:01:42,532 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:01:42,532 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:01:42,537 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:01:42,800 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/inojqhlt-media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:42,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:43,397 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media +2025-01-06 21:01:43,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:01:43,686 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/xd62vwum-media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:01:43,858 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:44,398 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:44,859 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:45,859 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:46,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:47,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:48,185 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:48,399 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:48,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:49,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:50,400 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:50,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:51,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:52,401 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:01:52,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:53,497 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 21:01:53,498 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:01:53,498 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:01:53,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:01:53,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:54,401 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:54,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:55,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:56,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:57,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:58,403 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:01:58,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:01:58,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:01:59,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:00,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:01,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:02,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:03,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:03,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:04,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:05,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:06,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:07,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:08,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:08,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:08,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:08,918 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:09,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:10,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:11,863 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:12,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:13,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:14,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:14,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:15,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:16,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:17,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:18,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:19,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:19,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:20,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:21,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:22,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:23,507 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:02:23,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:23,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:23,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:24,179 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:02:24,179 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 with policy now +2025-01-06 21:02:24,182 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:02:24,183 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:02:24,183 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:02:24,184 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:02:24,448 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/dnchds9h-media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,720 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/fbs9frno-media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:02:24,868 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:25,201 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:25,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:26,411 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:26,868 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:27,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:28,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:29,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:30,413 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:30,448 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:30,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:31,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:32,413 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:02:32,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:32,675 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/zupl7yxz-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:02:32,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:33,372 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:02:33,372 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:02:33,373 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:02:33,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:02:33,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:34,414 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:34,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:35,825 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:35,871 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:36,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:37,872 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:38,415 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:02:38,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:38,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:38,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:39,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:40,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:41,009 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:41,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:42,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:43,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:44,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:45,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:46,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:46,873 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:47,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:48,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:49,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:50,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:51,855 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:51,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:52,874 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:53,499 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:02:53,852 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:02:53,852 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:02:53,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:54,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:55,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:56,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:56,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:02:57,875 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:58,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:02:59,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:00,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:01,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:02,778 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:02,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:03,882 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:04,249 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:03:04,249 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 with policy now +2025-01-06 21:03:04,251 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:03:04,252 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:03:04,252 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:03:04,252 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,424 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:03:04,503 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/nnw7v8qe-media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,796 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/cr1k5tmp-media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:03:04,879 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:05,884 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:06,425 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:06,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:07,889 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:08,425 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:08,621 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:08,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:08,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:08,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:09,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:10,884 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:11,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:12,427 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:12,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:13,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:13,885 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:14,886 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:15,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:16,891 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:17,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:18,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:18,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:19,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:20,887 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:21,891 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:22,888 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:23,499 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:03:23,856 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:23,856 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:23,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:23,973 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:24,899 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:25,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:26,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:27,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:28,890 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:29,865 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:29,892 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:30,897 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:31,893 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:32,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:33,895 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:34,894 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:35,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:35,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:36,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:37,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:38,861 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:38,861 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:38,909 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:39,899 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:40,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:41,147 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:03:41,147 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:03:41,148 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:41,148 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:03:41,436 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:03:41,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:03:41,680 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/uj9tdk34-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:03:41,903 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:42,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:42,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:43,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:44,437 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:44,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:45,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:46,438 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:46,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:47,004 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:47,902 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:48,901 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:49,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:50,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:51,905 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:52,862 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:52,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:53,500 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:03:53,853 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:03:53,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:03:53,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:54,441 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:03:54,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:55,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:56,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:57,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:57,995 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:03:58,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:03:59,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:00,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:01,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:02,906 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:03,862 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:03,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:04,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:05,907 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:06,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:07,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:08,854 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:08,854 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:08,908 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:08,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:09,909 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:10,446 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:10,910 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:11,913 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:12,447 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:12,917 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:13,230 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:04:13,230 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 with policy now +2025-01-06 21:04:13,243 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:04:13,243 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:13,243 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:13,244 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:04:13,492 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/8e25x4z2-media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,760 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/wy76zwe0-media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:04:13,923 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:14,448 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:14,866 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:14,917 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:15,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:16,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:17,924 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:18,449 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:18,922 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:19,921 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:20,409 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:20,450 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:20,928 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:21,927 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:22,924 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:23,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:04:23,860 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:23,860 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:23,922 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:24,200 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:24,200 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:24,201 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:24,451 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:04:24,452 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:24,930 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:25,929 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:26,117 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:26,452 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:26,938 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:27,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:28,940 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:29,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:30,454 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:30,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:31,591 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:31,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:32,931 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:33,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:34,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:35,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:36,867 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:36,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:37,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:38,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:38,857 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:38,932 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:39,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:40,457 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:40,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:41,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:41,978 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:42,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:43,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:44,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:45,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:46,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:47,867 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:47,933 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:48,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:49,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:50,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:51,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:52,870 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:52,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:53,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:04:53,857 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:04:53,857 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:04:53,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:54,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:55,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:56,939 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:57,669 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:04:57,669 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 with policy now +2025-01-06 21:04:57,674 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:04:57,674 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:04:57,674 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:04:57,675 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:04:57,937 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/w0jcvthq-media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:57,938 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:04:58,478 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:58,479 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval +2025-01-06 21:04:58,706 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:04:58,770 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/05y7ayc2-media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:04:58,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:04:59,943 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:00,949 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:01,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:02,479 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:02,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:03,876 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:03,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:04,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:05,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:06,481 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:06,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:07,952 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:08,864 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:05:08,864 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:05:08,951 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:08,989 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:09,957 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:10,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:11,959 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:12,951 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:13,953 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:14,878 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:14,959 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:15,956 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:16,955 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:17,958 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:18,970 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:19,882 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:19,965 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:20,968 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:21,965 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:22,969 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:23,501 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:05:23,870 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:05:23,871 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:05:23,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:23,998 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:05:24,011 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:05:24,268 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:05:24,474 INFO wandb-upload_2:2117937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7ygkp7hl +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:24,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media +2025-01-06 21:05:24,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval +2025-01-06 21:05:24,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table +2025-01-06 21:05:24,679 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:05:24,687 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:05:24,959 INFO SenderThread:2117937 [sender.py:send_request_log_artifact():1525] logged artifact run-3d3i07nt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM3MDQwNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2NTMxOQ==', 'latestArtifact': None}} +2025-01-06 21:05:24,959 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:05:24,961 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:24,961 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now +2025-01-06 21:05:24,961 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:24,961 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,962 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,962 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:24,962 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:24,963 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send():379] send: telemetry +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,963 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,963 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,964 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,964 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,964 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,965 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,965 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,965 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,965 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,966 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,966 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,966 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,967 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,967 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,967 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,967 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:24,968 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:24,968 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:05:25,205 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/nesakpao-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:25,210 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/fadlkzzd-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:25,369 INFO wandb-upload_2:2117937 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwont73sm +2025-01-06 21:05:25,449 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/jmvf3due-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:25,487 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:25,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval +2025-01-06 21:05:25,733 INFO SenderThread:2117937 [sender.py:send_request_log_artifact():1525] logged artifact run-3d3i07nt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM3MDU1Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2NTQwMQ==', 'latestArtifact': None}} +2025-01-06 21:05:25,733 DEBUG SenderThread:2117937 [sender.py:send():379] send: files +2025-01-06 21:05:25,733 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json with policy now +2025-01-06 21:05:25,733 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:25,734 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:25,734 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:25,908 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/g94hhcqb-media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:25,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:26,157 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/h17ehwwa-media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:26,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:26,488 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:26,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:27,963 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:28,963 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:29,962 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:30,005 DEBUG SenderThread:2117937 [sender.py:send():379] send: exit +2025-01-06 21:05:30,005 INFO SenderThread:2117937 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_exit():588] handling runtime: 276 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_exit():594] send defer +2025-01-06 21:05:30,006 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,006 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 21:05:30,006 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 21:05:30,006 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 1 +2025-01-06 21:05:30,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 21:05:30,007 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,007 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 21:05:30,007 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 2 +2025-01-06 21:05:30,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 21:05:30,007 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 21:05:30,008 DEBUG SystemMonitor:2117937 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 21:05:30,007 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 21:05:30,008 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 21:05:30,059 INFO HandlerThread:2117937 [interfaces.py:finish():200] Joined network monitor +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 3 +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send():379] send: stats +2025-01-06 21:05:30,060 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,060 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 21:05:30,060 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 21:05:30,060 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 4 +2025-01-06 21:05:30,060 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:30,061 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 21:05:30,489 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:32,490 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:36,277 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:36,278 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,278 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 21:05:36,278 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 5 +2025-01-06 21:05:36,278 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:36,278 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:36,278 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 21:05:36,279 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,279 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 21:05:36,280 DEBUG SenderThread:2117937 [sender.py:send():379] send: history +2025-01-06 21:05:36,280 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:05:36,281 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:36,281 DEBUG SenderThread:2117937 [sender.py:send():379] send: summary +2025-01-06 21:05:36,285 INFO SenderThread:2117937 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:05:36,285 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,285 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 21:05:36,286 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 6 +2025-01-06 21:05:36,286 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,286 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 21:05:36,286 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,286 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 21:05:36,289 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:05:36,480 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 7 +2025-01-06 21:05:36,481 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:36,481 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 21:05:36,481 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:36,481 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 21:05:36,492 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:05:36,492 INFO Thread-12 :2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:37,007 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:40,281 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 8 +2025-01-06 21:05:40,281 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:40,281 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,282 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 21:05:40,282 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,282 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 21:05:40,282 INFO SenderThread:2117937 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 21:05:40,284 INFO SenderThread:2117937 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 21:05:40,286 INFO SenderThread:2117937 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 21:05:40,300 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 9 +2025-01-06 21:05:40,300 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,301 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 21:05:40,301 DEBUG SenderThread:2117937 [sender.py:send():379] send: artifact +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 21:05:40,489 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 21:05:40,489 INFO SenderThread:2117937 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 21:05:40,493 INFO SenderThread:2117937 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:40,493 INFO SenderThread:2117937 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml config.yaml +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json wandb-summary.json +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt requirements.txt +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-metadata.json wandb-metadata.json +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log output.log +2025-01-06 21:05:40,494 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/diff.patch diff.patch +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 media/videos/eval/0_eval_0_3bab73a244800e000493.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 media/videos/eval/4_eval_4_d51249e60c287ad366a8.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 media/videos/eval/2_eval_2_559e440a12c1c69e9e1a.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 media/videos/eval/3_eval_3_9a07183c07b2b4777782.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 media/videos/eval/1_eval_1_9cbaf6111e2b85cffc2c.mp4 +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json media/table/eval/total_success_rate_ci_6_b2a679debf83a2b3c3cf.table.json +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json +2025-01-06 21:05:40,498 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 10 +2025-01-06 21:05:40,499 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,503 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 21:05:40,504 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,505 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 21:05:40,505 INFO SenderThread:2117937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:05:40,675 INFO wandb-upload_0:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/config.yaml +2025-01-06 21:05:40,707 INFO wandb-upload_2:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/wandb-summary.json +2025-01-06 21:05:40,728 INFO wandb-upload_1:2117937 [upload_job.py:push():130] Uploaded file /tmp/tmp8zvqyydjwandb/mnxincra-events.out.tfevents.1736215253.leonmkim-ROG-Strix-G15CS-G15CS.2117773.0 +2025-01-06 21:05:40,731 INFO wandb-upload_4:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/output.log +2025-01-06 21:05:40,733 INFO wandb-upload_3:2117937 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/files/requirements.txt +2025-01-06 21:05:40,934 INFO Thread-11 (_thread_body):2117937 [sender.py:transition_state():614] send defer: 11 +2025-01-06 21:05:40,934 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,934 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 21:05:40,934 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,935 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 21:05:40,935 INFO SenderThread:2117937 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 12 +2025-01-06 21:05:40,936 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:40,936 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 21:05:40,936 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 21:05:40,936 INFO SenderThread:2117937 [file_stream.py:finish():601] file stream finish called +2025-01-06 21:05:41,008 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [file_stream.py:finish():605] file stream finish is done +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 13 +2025-01-06 21:05:41,162 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,162 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:41,162 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 21:05:41,162 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 21:05:41,162 INFO SenderThread:2117937 [sender.py:transition_state():614] send defer: 14 +2025-01-06 21:05:41,162 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:05:41,162 INFO HandlerThread:2117937 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send():379] send: final +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send():379] send: footer +2025-01-06 21:05:41,163 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: defer +2025-01-06 21:05:41,163 INFO SenderThread:2117937 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 21:05:41,163 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,164 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 21:05:41,164 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,164 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: server_info +2025-01-06 21:05:41,166 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:05:41,166 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 21:05:41,167 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 21:05:41,167 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:05:41,229 DEBUG SenderThread:2117937 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 21:05:41,230 INFO MainThread:2117937 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 21:05:41,231 DEBUG HandlerThread:2117937 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 21:05:41,231 INFO HandlerThread:2117937 [handler.py:finish():882] shutting down handler +2025-01-06 21:05:42,167 INFO WriterThread:2117937 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [sender.py:finish():1615] shutting down sender +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:05:42,230 INFO SenderThread:2117937 [file_pusher.py:join():175] waiting for file pusher diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug.log b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d748241179217175b8a4f06bb6fc75fb673fe319 --- /dev/null +++ b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Configure stats pid to 2117773 +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/settings +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug.log +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/logs/debug-internal.log +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():569] calling init triggers +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt'} +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():619] starting backend +2025-01-06 21:00:53,090 INFO MainThread:2117773 [wandb_init.py:init():623] setting up manager +2025-01-06 21:00:53,096 INFO MainThread:2117773 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 21:00:53,097 INFO MainThread:2117773 [wandb_init.py:init():631] backend started and connected +2025-01-06 21:00:53,111 INFO MainThread:2117773 [wandb_init.py:init():720] updated telemetry +2025-01-06 21:00:53,117 INFO MainThread:2117773 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 21:00:53,401 INFO MainThread:2117773 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 21:00:53,441 INFO MainThread:2117773 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 21:00:53,441 INFO MainThread:2117773 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 21:00:53,852 INFO MainThread:2117773 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 21:00:53,853 INFO MainThread:2117773 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 21:00:53,854 INFO MainThread:2117773 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3d3i07nt/tb, True +2025-01-06 21:00:58,766 INFO MainThread:2117773 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 21:05:42,231 WARNING MsgRouterThr:2117773 [router.py:message_loop():77] message_loop has been closed diff --git a/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4607dd34ac20900df8843d6241072fbf4e7afc71 Binary files /dev/null and b/3d3i07nt/wandb/run-20250106_210053-3d3i07nt/run-3d3i07nt.wandb differ diff --git a/7u1dchof/episode_rosbags/episode_0_2025-01-07-21-50-00.bag b/7u1dchof/episode_rosbags/episode_0_2025-01-07-21-50-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..6d8b14dc14bcf19459ba2063c674ef21526d3fc9 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_0_2025-01-07-21-50-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6684154829aeea64f14e2c6e1e000482eb934868cb7efc6cb9ed39111f14c7dc +size 1411371628 diff --git a/7u1dchof/episode_rosbags/episode_10_2025-01-07-21-57-39.bag b/7u1dchof/episode_rosbags/episode_10_2025-01-07-21-57-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..cf6d33a5a6ffcc1b30fd554c3d86da0f8bd6b069 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_10_2025-01-07-21-57-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab81275d1a8a61c43e54fe0923cea7101c7103325c6425f596770447ab15abb +size 1417088125 diff --git a/7u1dchof/episode_rosbags/episode_13_2025-01-07-22-00-06.bag b/7u1dchof/episode_rosbags/episode_13_2025-01-07-22-00-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..8cf9baabe38b99c36f845cf3a472a8c900251c5d --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_13_2025-01-07-22-00-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d80b9c7f6d99be929b50a87d7922cb89b55d470622a8f07e85b63d3851c938ed +size 1414528981 diff --git a/7u1dchof/episode_rosbags/episode_2_2025-01-07-21-51-32.bag b/7u1dchof/episode_rosbags/episode_2_2025-01-07-21-51-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..0d5f6ae76643e5af9c18164422d1e084a9112311 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_2_2025-01-07-21-51-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3c063d566d47db5d4ed41f0b32b412fd8336050e7faded2e2cb8e7dfbfc6cf6 +size 1416554458 diff --git a/7u1dchof/episode_rosbags/episode_7_2025-01-07-21-55-24.bag b/7u1dchof/episode_rosbags/episode_7_2025-01-07-21-55-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..c1b73792e05038901c2d4822a4b33a30717d0951 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_7_2025-01-07-21-55-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59c4e126219166ce4acc6225f553aa598b7d4fd7ca814b1040dee38dc1cc9cc6 +size 1407114267 diff --git a/8ycpgp3n/episode_rosbags/episode_2_2025-01-06-17-09-45.bag b/8ycpgp3n/episode_rosbags/episode_2_2025-01-06-17-09-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..bbdeae3aac5c4d02c2c17673cdcc582f860314bf --- /dev/null +++ b/8ycpgp3n/episode_rosbags/episode_2_2025-01-06-17-09-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123a27701b9018881bebab208778c110ef94fd4e873ebbd667adc2638fe11f96 +size 1411473015 diff --git a/8ycpgp3n/episode_rosbags/episode_3_2025-01-06-17-10-36.bag b/8ycpgp3n/episode_rosbags/episode_3_2025-01-06-17-10-36.bag new file mode 100644 index 0000000000000000000000000000000000000000..425712f48738ea2d706d04258f2ff072ec78c0cd --- /dev/null +++ b/8ycpgp3n/episode_rosbags/episode_3_2025-01-06-17-10-36.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ca4a8e489ba354ee7d00ab8211f0fb1d8b36a81235dfb075ae3b293cc75c566 +size 1409907240 diff --git a/fnq3cpdm/.hydra/config.yaml b/fnq3cpdm/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/fnq3cpdm/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/fnq3cpdm/.hydra/hydra.yaml b/fnq3cpdm/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4a3b7bfac5de9ddd246f89103575feae9d639f8 --- /dev/null +++ b/fnq3cpdm/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/fnq3cpdm/.hydra/overrides.yaml b/fnq3cpdm/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/fnq3cpdm/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/fnq3cpdm/episode_rosbags/aligned_depth_to_color_K.npy b/fnq3cpdm/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/fnq3cpdm/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/fnq3cpdm/episode_rosbags/color_K.npy b/fnq3cpdm/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/fnq3cpdm/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/fnq3cpdm/episode_rosbags/episode_2_2025-01-06-17-15-32.bag b/fnq3cpdm/episode_rosbags/episode_2_2025-01-06-17-15-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..6f6a1c35e68260e01452ffa9d2ca69838a4c95fa --- /dev/null +++ b/fnq3cpdm/episode_rosbags/episode_2_2025-01-06-17-15-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4e95a8e44850f86fe421abc591aa3c1b725c292ec7eb00908e7f2a6e9f63236 +size 1409927604 diff --git a/fnq3cpdm/eval_video/0_eval.mp4 b/fnq3cpdm/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cc235de70076750f45193e22d54b724485c243e3 --- /dev/null +++ b/fnq3cpdm/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa73dbea3eafd7c0a8332ed5db27a0081a69b55679c6d6c14f4a7dc10068af4 +size 1188581 diff --git a/fnq3cpdm/eval_video/1_eval.mp4 b/fnq3cpdm/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05ea03f980d1dd1070187796d87ebfb1e6805dc4 --- /dev/null +++ b/fnq3cpdm/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:796f3c4bcdaca1535e57979e88124547ee0706a1e40b1aca2f95a0841dfeb76d +size 1133702 diff --git a/fnq3cpdm/eval_video/2_eval.mp4 b/fnq3cpdm/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ef1f3b549098afe77cbe17a6e764aed0b583baa --- /dev/null +++ b/fnq3cpdm/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb7fe7e446267787ce7d5761b72998d7e2ad20da21d63319be3d652f34be2d1 +size 1142872 diff --git a/fnq3cpdm/eval_video/3_eval.mp4 b/fnq3cpdm/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0098c9ad7030f7ad81760a0e467150007fe388de --- /dev/null +++ b/fnq3cpdm/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f7da30cdf24122b19f9b6bc446f2a1145c586c3775fe60d3229fd63da47e6f +size 1195740 diff --git a/fnq3cpdm/eval_video/4_eval.mp4 b/fnq3cpdm/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..954e0557ed514f777de2ea3851b7b2eb3dc397ba --- /dev/null +++ b/fnq3cpdm/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e609f284fc921a805bf4065510cc2ad73228369281ef4f5655e5bf4dc2ca5ad +size 1209249 diff --git a/fnq3cpdm/tb/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 b/fnq3cpdm/tb/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 new file mode 100644 index 0000000000000000000000000000000000000000..d08053843df88fee627fe6735d77578fcd814a66 --- /dev/null +++ b/fnq3cpdm/tb/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5be2368d7702d8591f07795cb9a19c9229cd88b58d129f1f221bfebf8ef3a2 +size 6539 diff --git a/fnq3cpdm/wandb/debug-internal.log b/fnq3cpdm/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b7320f655c112c6fd90471a13518f79c65f1b585 --- /dev/null +++ b/fnq3cpdm/wandb/debug-internal.log @@ -0,0 +1,830 @@ +2025-01-06 17:12:51,047 INFO StreamThr :2086753 [internal.py:wandb_internal():85] W&B internal server running at pid: 2086753, started at: 2025-01-06 17:12:51.047338 +2025-01-06 17:12:51,048 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:12:51,060 INFO WriterThread:2086753 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb +2025-01-06 17:12:51,061 DEBUG SenderThread:2086753 [sender.py:send():379] send: header +2025-01-06 17:12:51,068 DEBUG SenderThread:2086753 [sender.py:send():379] send: run +2025-01-06 17:12:51,371 INFO SenderThread:2086753 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files +2025-01-06 17:12:51,372 INFO SenderThread:2086753 [sender.py:_start_run_threads():1188] run started: fnq3cpdm with start time 1736201571.048813 +2025-01-06 17:12:51,389 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:12:51,390 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:12:51,463 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:12:51,468 DEBUG HandlerThread:2086753 [system_info.py:__init__():26] System info init +2025-01-06 17:12:51,468 DEBUG HandlerThread:2086753 [system_info.py:__init__():41] System info init done +2025-01-06 17:12:51,468 INFO HandlerThread:2086753 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:12:51,469 INFO SystemMonitor:2086753 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:12:51,469 INFO HandlerThread:2086753 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:12:51,469 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:12:51,470 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:12:51,470 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:12:51,471 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:12:51,471 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:12:51,481 DEBUG HandlerThread:2086753 [system_info.py:probe():152] Probing system +2025-01-06 17:12:51,483 DEBUG HandlerThread:2086753 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_info.py:probe():200] Probing system done +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:12:51.481677', 'startedAt': '2025-01-06T22:12:51.039379', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.493125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.958, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.425, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 320.86563873291016}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:12:51,488 INFO HandlerThread:2086753 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:12:51,488 INFO HandlerThread:2086753 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:12:51,488 DEBUG HandlerThread:2086753 [system_info.py:_save_code():44] Saving code +2025-01-06 17:12:51,489 DEBUG HandlerThread:2086753 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:12:51,489 DEBUG HandlerThread:2086753 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:12:51,559 DEBUG HandlerThread:2086753 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:12:51,559 INFO HandlerThread:2086753 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:12:51,562 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:12:51,562 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:12:51,563 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:12:51,563 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:12:51,800 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/jqla7lri-wandb-metadata.json +2025-01-06 17:12:51,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:12:51,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:12:51,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:12:51,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:12:51,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:51,814 INFO HandlerThread:2086753 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/tb" + save: true +} +control { + relay_id: "83c02b91baf3499fb941f0c7d9259a5b" +} +_info { + stream_id: "fnq3cpdm" +} + +2025-01-06 17:12:51,814 DEBUG HandlerThread:2086753 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:12:51,843 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/1qaw1h0w-diff.patch +2025-01-06 17:12:51,862 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/rq6hd07z-code/FISH/eval_robot.py +2025-01-06 17:12:51,947 DEBUG SenderThread:2086753 [sender.py:send():379] send: telemetry +2025-01-06 17:12:51,947 DEBUG SenderThread:2086753 [sender.py:send():379] send: tbrecord +2025-01-06 17:12:51,948 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:12:51,948 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 with policy live +2025-01-06 17:12:52,131 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/a0bptdxd-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH +2025-01-06 17:12:52,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:53,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:54,372 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:12:54,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:55,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:56,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:56,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:57,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:58,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:59,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:00,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:01,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:02,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:02,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:03,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:04,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:05,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:06,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:06,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:07,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:07,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:08,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:09,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:10,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:11,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:12,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:13,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:13,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:14,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:15,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:16,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:17,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:18,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:18,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:19,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:20,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:21,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:21,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:21,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:22,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:23,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:23,890 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:24,380 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:13:24,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:25,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:26,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:27,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:28,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:29,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:29,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:30,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:31,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:32,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:33,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:34,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:34,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:35,087 DEBUG SenderThread:2086753 [sender.py:send():379] send: config +2025-01-06 17:13:35,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:36,384 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:36,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:36,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:37,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:38,384 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:38,845 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:39,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:39,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:40,385 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:40,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:41,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:42,385 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:42,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:43,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:44,386 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:44,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:45,569 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:45,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:46,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:47,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:48,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:49,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:50,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:50,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:51,471 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:13:51,472 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:13:51,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:51,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:51,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:52,388 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:52,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:53,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:54,389 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:54,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:55,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:56,389 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:56,445 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:56,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:57,390 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:13:57,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:58,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:59,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:00,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:01,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:01,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:02,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:03,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:04,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:05,822 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:06,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:06,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:06,941 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:07,822 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:08,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:09,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:10,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:11,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:12,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:12,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:13,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:14,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:15,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:16,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:17,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:17,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:18,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:19,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:20,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:21,472 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:14:21,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:21,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:21,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:22,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:22,895 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:23,106 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:14:23,106 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 with policy now +2025-01-06 17:14:23,106 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:14:23,107 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:14:23,108 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:14:23,108 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:14:23,370 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/wdfckj65-media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,397 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media +2025-01-06 17:14:23,692 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/nrtcly84-media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:24,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:24,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:25,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:26,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:27,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:28,399 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:28,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:28,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:29,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:30,400 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:30,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:31,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:32,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:33,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:34,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:34,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:35,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:36,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:36,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:37,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:38,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:39,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:39,963 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:40,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:41,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:42,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:43,404 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:14:43,643 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/7pvsuix8-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:14:43,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:44,405 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:44,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:45,737 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:45,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:46,405 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:46,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:47,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:48,406 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:48,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:49,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:50,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:50,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:51,473 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:14:51,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:51,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:51,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:52,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:53,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:54,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:55,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:55,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:56,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:57,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:58,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:59,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:00,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:01,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:01,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:02,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:03,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:04,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:05,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:06,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:06,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:06,906 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:07,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:08,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:09,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:10,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:11,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:12,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:12,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:13,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:14,415 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:14,493 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:15:14,493 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 with policy now +2025-01-06 17:15:14,496 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:15:14,496 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:15:14,496 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:15:14,496 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:15:14,758 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/dozw7j6p-media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:14,834 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:15:15,733 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/47o71szn-media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:15,833 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:16,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:16,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:17,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:17,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:18,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:19,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:20,417 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:20,835 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:21,473 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:15:21,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:21,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:21,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:22,418 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:22,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:22,889 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:23,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:24,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:25,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:26,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:27,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:28,478 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:28,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:29,272 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:15:29,272 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:15:29,273 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:15:29,421 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:15:29,422 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:15:29,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:30,422 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:30,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:31,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:32,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:33,683 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:33,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:34,423 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:34,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:35,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:36,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:36,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:37,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:38,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:38,939 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:39,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:40,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:41,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:42,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:43,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:44,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:44,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:45,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:46,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:47,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:48,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:49,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:50,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:50,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:51,474 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:15:51,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:51,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:51,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:52,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:53,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:54,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:55,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:55,915 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:56,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:57,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:58,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:59,842 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:00,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:00,505 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:16:00,505 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 with policy now +2025-01-06 17:16:00,507 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:16:00,508 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:16:00,508 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:16:00,508 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:16:00,758 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/djjo0upu-media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:00,847 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:01,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:16:01,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:01,433 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:16:01,530 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:01,705 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/0hj6nayy-media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:01,846 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:02,433 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:02,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:03,855 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:04,859 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:05,860 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:06,434 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:06,739 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:06,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:06,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:06,856 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:07,862 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:08,435 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:08,865 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:09,862 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:10,859 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:11,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:11,860 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:12,863 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:13,861 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:14,863 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:15,866 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:16,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:17,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:17,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:18,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:19,873 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:20,869 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:21,474 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:16:21,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:21,817 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:21,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:22,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:22,948 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:23,868 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:24,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:25,870 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:26,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:27,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:28,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:28,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:29,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:30,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:31,874 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:32,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:33,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:34,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:34,879 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:35,878 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:36,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:36,823 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:36,878 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:37,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:38,444 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:16:38,689 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/9bxsd8t8-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:16:38,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:39,139 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:16:39,140 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:16:39,140 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:16:39,445 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:16:39,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:40,445 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:40,465 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:40,885 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:41,886 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:42,880 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:43,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:44,446 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:44,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:45,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:46,405 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:46,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:47,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:48,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:49,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:50,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:51,475 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:16:51,475 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:51,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:51,817 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:51,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:52,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:53,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:54,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:55,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:56,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:56,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:57,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:58,451 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:58,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:59,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:00,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:01,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:01,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:02,452 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:02,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:03,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:04,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:05,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:06,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:06,815 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:06,887 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:06,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:07,888 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:08,887 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:09,890 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:10,455 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:10,610 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:17:10,610 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 with policy now +2025-01-06 17:17:10,613 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:17:10,614 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:10,614 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:10,614 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:10,891 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/kf7o4uj8-media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:10,892 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:11,455 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:11,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:11,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:17:11,742 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/zif8y681-media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:11,893 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:12,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:12,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:12,894 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:13,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:13,894 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:14,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:15,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:16,457 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:16,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:17,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:17,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:18,458 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:18,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:19,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:20,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:21,475 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:17:21,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:21,819 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:21,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:22,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:23,192 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:23,193 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:23,193 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:23,193 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:23,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:17:23,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:23,903 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:24,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:24,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:25,904 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:26,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:27,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:28,461 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:28,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:29,034 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:29,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:30,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:31,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:32,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:33,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:34,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:34,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:35,908 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:36,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:36,815 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:36,909 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:37,909 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:38,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:39,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:39,971 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:40,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:41,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:42,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:43,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:44,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:45,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:45,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:46,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:47,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:48,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:49,914 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:50,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:50,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:51,476 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:17:51,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:51,816 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:51,912 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:52,917 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:53,918 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:54,470 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:54,923 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:55,177 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:17:55,177 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 with policy now +2025-01-06 17:17:55,180 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:17:55,181 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:55,181 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:55,181 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:55,431 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/1jvb2v6t-media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,470 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:55,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:17:55,735 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/adsnt5y9-media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,924 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:56,206 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:56,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:56,929 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:57,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:57,939 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:58,947 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:59,937 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:00,472 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:00,936 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:01,421 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:01,952 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:02,473 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:02,950 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:03,947 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:04,177 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:18:04,177 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:18:04,396 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval +2025-01-06 17:18:04,649 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/9wen6loj-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:04,683 INFO wandb-upload_1:2086753 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbuwzuhnv +2025-01-06 17:18:04,756 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:18:04,767 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:18:04,792 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/iicmhn41-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:04,942 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:05,091 INFO SenderThread:2086753 [sender.py:send_request_log_artifact():1525] logged artifact run-fnq3cpdm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkxNDcxNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwOTY0MA==', 'latestArtifact': None}} +2025-01-06 17:18:05,091 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:18:05,091 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:18:05,091 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:05,092 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,093 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,093 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:05,094 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,097 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,098 DEBUG SenderThread:2086753 [sender.py:send():379] send: telemetry +2025-01-06 17:18:05,098 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,099 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,099 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,101 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,101 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,103 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,103 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,105 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,105 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,105 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,105 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,107 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,107 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,109 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,110 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,110 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,110 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,112 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,112 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,114 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,114 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,115 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,115 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:18:05,338 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/904i8vbd-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:05,471 INFO wandb-upload_0:2086753 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1n0gr8op +2025-01-06 17:18:05,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:05,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:05,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:05,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval +2025-01-06 17:18:05,942 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:06,020 INFO SenderThread:2086753 [sender.py:send_request_log_artifact():1525] logged artifact run-fnq3cpdm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkxNDc0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwOTY0Mg==', 'latestArtifact': None}} +2025-01-06 17:18:06,020 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:18:06,020 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json with policy now +2025-01-06 17:18:06,021 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:06,021 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:06,021 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:06,206 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/7p212ht7-media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:06,450 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/k8d8b30d-media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:06,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:06,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:06,491 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:06,537 DEBUG SenderThread:2086753 [sender.py:send():379] send: exit +2025-01-06 17:18:06,537 INFO SenderThread:2086753 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_exit():588] handling runtime: 315 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_exit():594] send defer +2025-01-06 17:18:06,538 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,538 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:18:06,538 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:18:06,538 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:18:06,539 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,539 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:18:06,539 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:18:06,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:18:06,539 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:18:06,539 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:18:06,540 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:18:06,540 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:18:06,611 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,611 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:18:06,611 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:18:06,611 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:18:06,612 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,612 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:18:06,612 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,612 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:18:06,612 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:18:06,612 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,612 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:18:07,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:08,476 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:13,248 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:13,248 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:13,249 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,250 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:18:13,250 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:13,250 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,250 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:18:13,251 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:13,251 DEBUG SenderThread:2086753 [sender.py:send():379] send: summary +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:13,255 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:18:13,255 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,255 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:18:13,256 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,256 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:18:13,259 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:13,411 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:18:13,411 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,411 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:18:13,411 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,411 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:18:13,478 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:13,478 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:18:13,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:18:14,574 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:14,574 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:14,574 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:18:14,574 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:18:14,577 INFO SenderThread:2086753 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:18:14,579 INFO SenderThread:2086753 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:18:14,596 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:18:14,596 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:14,596 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:18:14,597 DEBUG SenderThread:2086753 [sender.py:send():379] send: artifact +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'versionIndex': 422}}} +2025-01-06 17:18:14,795 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml config.yaml +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json wandb-summary.json +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt requirements.txt +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log output.log +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch diff.patch +2025-01-06 17:18:15,483 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:18:15,488 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:18:15,489 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:18:15,491 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,491 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:18:15,491 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,491 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:18:15,491 INFO SenderThread:2086753 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:18:15,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:15,540 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:15,675 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:18:15,677 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:15,690 INFO wandb-upload_4:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:15,724 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/ncgpht3y-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:15,744 INFO wandb-upload_3:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt +2025-01-06 17:18:15,945 INFO Thread-11 (_thread_body):2086753 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:18:15,945 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,945 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:18:15,945 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,945 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:18:15,945 INFO SenderThread:2086753 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:18:15,946 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:18:15,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,946 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:18:15,947 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,947 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:18:15,947 INFO SenderThread:2086753 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:18:16,181 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:16,181 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:18:16,181 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:18:16,181 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:16,181 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send():379] send: final +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send():379] send: footer +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:16,182 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:18:16,182 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:18:16,183 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:18:16,184 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:18:16,185 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:16,234 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:16,234 INFO MainThread:2086753 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:18:16,235 INFO MainThread:2086753 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:18:16,235 INFO MainThread:2086753 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:18:16,235 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:18:16,235 INFO HandlerThread:2086753 [handler.py:finish():882] shutting down handler +2025-01-06 17:18:17,183 INFO WriterThread:2086753 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [sender.py:finish():1615] shutting down sender +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [file_pusher.py:join():175] waiting for file pusher diff --git a/fnq3cpdm/wandb/debug.log b/fnq3cpdm/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e9797b08153ba5a3999513417dcbfd7c7a88e3a6 --- /dev/null +++ b/fnq3cpdm/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Configure stats pid to 2086588 +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/settings +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug.log +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug-internal.log +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm'} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():619] starting backend +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():623] setting up manager +2025-01-06 17:12:51,046 INFO MainThread:2086588 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:12:51,048 INFO MainThread:2086588 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:12:51,060 INFO MainThread:2086588 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:12:51,065 INFO MainThread:2086588 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:12:51,389 INFO MainThread:2086588 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:12:51,457 INFO MainThread:2086588 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:12:51,457 INFO MainThread:2086588 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:12:51,814 INFO MainThread:2086588 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:12:51,814 INFO MainThread:2086588 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/tb, True +2025-01-06 17:13:35,086 INFO MainThread:2086588 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:18:17,236 WARNING MsgRouterThr:2086588 [router.py:message_loop():77] message_loop has been closed diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c7d54713f7107dc668e68dc3ac1169b4aca9d569 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736201571 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json new file mode 100644 index 0000000000000000000000000000000000000000..56cc3c1978692983b638680b0a95dd37bd82990d --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[6, 15, 0.4, 0.14431723959407056, 0.7007466852263454, 0.19101621216754777, 0.6749166249879048, 0.21997335558769782, 0.6047569811134632, 0.2341548726494329, 0.5904298052010876]]} \ No newline at end of file diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cc235de70076750f45193e22d54b724485c243e3 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa73dbea3eafd7c0a8332ed5db27a0081a69b55679c6d6c14f4a7dc10068af4 +size 1188581 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05ea03f980d1dd1070187796d87ebfb1e6805dc4 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:796f3c4bcdaca1535e57979e88124547ee0706a1e40b1aca2f95a0841dfeb76d +size 1133702 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ef1f3b549098afe77cbe17a6e764aed0b583baa --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb7fe7e446267787ce7d5761b72998d7e2ad20da21d63319be3d652f34be2d1 +size 1142872 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0098c9ad7030f7ad81760a0e467150007fe388de --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f7da30cdf24122b19f9b6bc446f2a1145c586c3775fe60d3229fd63da47e6f +size 1195740 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..954e0557ed514f777de2ea3851b7b2eb3dc397ba --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e609f284fc921a805bf4065510cc2ad73228369281ef4f5655e5bf4dc2ca5ad +size 1209249 diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..073753848d67c3c191338164676c262ead452d77 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736201615.087399]: resetting environment +[INFO] [1736201615.090577]: cleared current plan +[INFO] [1736201615.091631]: moving to home +[INFO] [1736201616.093582]: reached home +[INFO] [1736201616.095195]: reset action history +[INFO] [1736201618.803245]: environment reset +Starting episode 0 +[INFO] [1736201618.805217]: resetting environment +[INFO] [1736201618.810042]: cleared current plan +[INFO] [1736201618.811204]: moving to home +[INFO] [1736201619.813263]: reached home +[INFO] [1736201619.814666]: reset action history +[INFO] [1736201622.522542]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736201631.438866]: resetting environment +[INFO] [1736201631.442151]: cleared current plan +[INFO] [1736201631.443062]: moving to home +[INFO] [1736201632.445223]: reached home +[INFO] [1736201632.446856]: reset action history +[INFO] [1736201635.153568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/episode_rosbags/episode_0_2025-01-06-17-13-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2086588.log +started roslaunch server http://158.130.50.37:39749/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2086856] +process[print_text-2]: started with pid [2086880] +process[rosbag_record-3]: started with pid [2086881] +started bagging! +[INFO] [1736201662.144044]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736201662.894848]: Stored episode 1. +[INFO] [1736201662.895197]: Saving video... +[INFO] [1736201663.095184]: Video saved! +Starting episode 1 +[INFO] [1736201663.107024]: resetting environment +[INFO] [1736201663.113370]: cleared current plan +[INFO] [1736201663.113607]: moving to home +[INFO] [1736201666.515793]: reached home +[INFO] [1736201666.516288]: reset action history +[INFO] [1736201669.222016]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201682.690148]: resetting environment +[INFO] [1736201682.693669]: cleared current plan +[INFO] [1736201682.693821]: moving to home +[INFO] [1736201683.695235]: reached home +[INFO] [1736201683.696032]: reset action history +[INFO] [1736201686.401939]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/episode_rosbags/episode_1_2025-01-06-17-14-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2086588.log +started roslaunch server http://158.130.50.37:42115/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2086993] +process[print_text-5]: started with pid [2086994] +process[rosbag_record-6]: started with pid [2086995] +started bagging! +[INFO] [1736201713.372514]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736201714.113651]: Stored episode 2. +[INFO] [1736201714.114191]: Saving video... +[INFO] [1736201714.472711]: Video saved! +Starting episode 2 +[INFO] [1736201714.497441]: resetting environment +[INFO] [1736201714.507338]: cleared current plan +[INFO] [1736201714.510030]: moving to home +[INFO] [1736201717.928229]: reached home +[INFO] [1736201717.929234]: reset action history +[INFO] [1736201720.697011]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201728.491571]: resetting environment +[INFO] [1736201728.498793]: cleared current plan +[INFO] [1736201728.499200]: moving to home +[INFO] [1736201729.503590]: reached home +[INFO] [1736201729.504052]: reset action history +[INFO] [1736201732.214655]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/episode_rosbags/episode_2_2025-01-06-17-15-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2086588.log +started roslaunch server http://158.130.50.37:36691/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2087152] +process[print_text-8]: started with pid [2087153] +process[rosbag_record-9]: started with pid [2087154] +started bagging! +[INFO] [1736201759.391929]: Storing episode... +For topic gripper_width: timestamp difference is 100781700 for nearest: 1736201759406890175 - target: 1736201759306108475 at idx 249 +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736201760.172183]: Stored episode 3. +[INFO] [1736201760.174767]: Saving video... +[INFO] [1736201760.463628]: Video saved! +Starting episode 3 +[INFO] [1736201760.510474]: resetting environment +[INFO] [1736201760.524010]: cleared current plan +[INFO] [1736201760.529340]: moving to home +[INFO] [1736201763.935751]: reached home +[INFO] [1736201763.942149]: reset action history +[INFO] [1736201766.736740]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201798.401341]: resetting environment +[INFO] [1736201798.413688]: cleared current plan +[INFO] [1736201798.415197]: moving to home +[INFO] [1736201799.422001]: reached home +[INFO] [1736201799.423695]: reset action history +[INFO] [1736201802.144341]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/episode_rosbags/episode_3_2025-01-06-17-16-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2086588.log +started roslaunch server http://158.130.50.37:41117/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2087293] +process[print_text-11]: started with pid [2087294] +process[rosbag_record-12]: started with pid [2087295] +started bagging! +For topic gripper_width: timestamp difference is -118989186 for nearest: 1736201817340132995 - target: 1736201817459122181 at idx 214 +For topic gripper_width: timestamp difference is -182307635 for nearest: 1736201821440133418 - target: 1736201821622441053 at idx 249 +For topic gripper_width: timestamp difference is -209831277 for nearest: 1736201828640133819 - target: 1736201828849965096 at idx 249 +[INFO] [1736201829.120774]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736201829.931736]: Stored episode 4. +[INFO] [1736201829.935081]: Saving video... +[INFO] [1736201830.579694]: Video saved! +Starting episode 4 +[INFO] [1736201830.616827]: resetting environment +[INFO] [1736201830.639219]: cleared current plan +[INFO] [1736201830.641388]: moving to home +[WARN] [1736201830.486719]: Plan exhausted +[WARN] [1736201830.530111]: Plan exhausted +[WARN] [1736201830.566890]: Plan exhausted +[WARN] [1736201830.607475]: Plan exhausted +[INFO] [1736201834.048679]: reached home +[INFO] [1736201834.057952]: reset action history +[INFO] [1736201836.858236]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201842.660263]: resetting environment +[INFO] [1736201842.688715]: cleared current plan +[INFO] [1736201842.689883]: moving to home +[INFO] [1736201843.692530]: reached home +[INFO] [1736201843.695424]: reset action history +[INFO] [1736201846.482419]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/episode_rosbags/episode_4_2025-01-06-17-17-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2086588.log +started roslaunch server http://158.130.50.37:41403/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2087456] +process[print_text-14]: started with pid [2087457] +process[rosbag_record-15]: started with pid [2087458] +started bagging! +[INFO] [1736201873.754632]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736201874.505315]: Stored episode 5. +[INFO] [1736201874.508213]: Saving video... +[INFO] [1736201875.148679]: Video saved! +[INFO] [1736201875.181487]: resetting environment +[INFO] [1736201875.203588]: cleared current plan +[INFO] [1736201875.204573]: moving to home +[WARN] [1736201875.086957]: Plan exhausted +[WARN] [1736201875.127861]: Plan exhausted +[WARN] [1736201875.166542]: Plan exhausted +[INFO] [1736201878.612410]: reached home +[INFO] [1736201878.624182]: reset action history +[INFO] [1736201881.419393]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..59736f8395d4e43ccc638d7999aa51261aa45b41 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:12:51.481677", + "startedAt": "2025-01-06T22:12:51.039379", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4123.493125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.958, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.425, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 320.86563873291016 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..02634c22e82e668490945513bd7f3b334dae2864 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "dfa73dbea3eafd7c0a8332ed5db27a0081a69b55679c6d6c14f4a7dc10068af4", "size": 1188581, "path": "media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4"}, "global_step": 664, "_timestamp": 1736201884.6543107, "_runtime": 313.60549759864807, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "796f3c4bcdaca1535e57979e88124547ee0706a1e40b1aca2f95a0841dfeb76d", "size": 1133702, "path": "media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4"}, "eval/num_success": 0.0, "episode": 15.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "acb7fe7e446267787ce7d5761b72998d7e2ad20da21d63319be3d652f34be2d1", "size": 1142872, "path": "media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "13f7da30cdf24122b19f9b6bc446f2a1145c586c3775fe60d3229fd63da47e6f", "size": 1195740, "path": "media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "0e609f284fc921a805bf4065510cc2ad73228369281ef4f5655e5bf4dc2ca5ad", "size": 1209249, "path": "media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://pc4a137ppzl13vu2f7m69icguktcj9s0odm52r30rnexi07986y6xz8wugurgumqz5zhlm57x7vzokkx08k9jxrbm9sy1hg2v2mw7i1nmh8z5b8k0nxkx2ok3nj266gz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://n2qma2j0y9huv27v5at588u942s9uabdqh6bdx9z8nfdagrqofebompwt86g515snj0vymtz3o89wm4v76y7krugv1rvb5x0038bneif9b2azifrmrejpl24234m9gzc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 6, "total_num_episodes": 15, "total_success_rate": 0.4, "total_success_rate_lb_0.01": 0.14431723959407056, "total_success_rate_ub_0.01": 0.7007466852263454, "total_success_rate_lb_0.025": 0.19101621216754777, "total_success_rate_ub_0.025": 0.6749166249879048, "total_success_rate_lb_0.05": 0.21997335558769782, "total_success_rate_ub_0.05": 0.6047569811134632, "total_success_rate_lb_0.1": 0.2341548726494329, "total_success_rate_ub_0.1": 0.5904298052010876, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c6b6ac0bad860bd9e38c22ff9f173518a735daa0da91b98646b3529225e7de1e", "size": 507, "artifact_path": "wandb-client-artifact://6478rd7sza84tnondmj848xi7xuew522gvjb9qqdk4u6vr6yvzybel007yr2xfe6ib55xnaj6nznvyqhxohp3mzom8od6rkfr7oyj3thmocs6p3ai0gpw1xt8g0d0ly4/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://n4gllkhffuukzb1nraggvc6a39enzxlqpiva44s7oecd58yxllrz3nodfubgs9q8cg7jb368dazl1x6rrb8eqhstze8ip17w6t9jxkdx77cif5p3jpv78ma9mc7nd4mx:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 315}, "eval/total_success_rate": 0.4000000059604645, "eval/total_success_rate_lb_0.01": 0.14431723952293396, "eval/total_success_rate_ub_0.01": 0.7007466554641724, "eval/total_success_rate_lb_0.025": 0.19101621210575104, "eval/total_success_rate_ub_0.025": 0.6749166250228882, "eval/total_success_rate_lb_0.05": 0.2199733555316925, "eval/total_success_rate_ub_0.05": 0.6047569513320923, "eval/total_success_rate_lb_0.1": 0.23415488004684448, "eval/total_success_rate_ub_0.1": 0.5904297828674316} \ No newline at end of file diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug-internal.log b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b7320f655c112c6fd90471a13518f79c65f1b585 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug-internal.log @@ -0,0 +1,830 @@ +2025-01-06 17:12:51,047 INFO StreamThr :2086753 [internal.py:wandb_internal():85] W&B internal server running at pid: 2086753, started at: 2025-01-06 17:12:51.047338 +2025-01-06 17:12:51,048 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:12:51,060 INFO WriterThread:2086753 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb +2025-01-06 17:12:51,061 DEBUG SenderThread:2086753 [sender.py:send():379] send: header +2025-01-06 17:12:51,068 DEBUG SenderThread:2086753 [sender.py:send():379] send: run +2025-01-06 17:12:51,371 INFO SenderThread:2086753 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files +2025-01-06 17:12:51,372 INFO SenderThread:2086753 [sender.py:_start_run_threads():1188] run started: fnq3cpdm with start time 1736201571.048813 +2025-01-06 17:12:51,389 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:12:51,390 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:12:51,463 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:12:51,468 DEBUG HandlerThread:2086753 [system_info.py:__init__():26] System info init +2025-01-06 17:12:51,468 DEBUG HandlerThread:2086753 [system_info.py:__init__():41] System info init done +2025-01-06 17:12:51,468 INFO HandlerThread:2086753 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:12:51,469 INFO SystemMonitor:2086753 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:12:51,469 INFO HandlerThread:2086753 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:12:51,469 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:12:51,470 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:12:51,470 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:12:51,471 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:12:51,471 INFO SystemMonitor:2086753 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:12:51,481 DEBUG HandlerThread:2086753 [system_info.py:probe():152] Probing system +2025-01-06 17:12:51,483 DEBUG HandlerThread:2086753 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_info.py:probe():200] Probing system done +2025-01-06 17:12:51,487 DEBUG HandlerThread:2086753 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:12:51.481677', 'startedAt': '2025-01-06T22:12:51.039379', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4123.493125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.958, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.425, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 320.86563873291016}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:12:51,488 INFO HandlerThread:2086753 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:12:51,488 INFO HandlerThread:2086753 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:12:51,488 DEBUG HandlerThread:2086753 [system_info.py:_save_code():44] Saving code +2025-01-06 17:12:51,489 DEBUG HandlerThread:2086753 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:12:51,489 DEBUG HandlerThread:2086753 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:12:51,559 DEBUG HandlerThread:2086753 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:12:51,559 INFO HandlerThread:2086753 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:12:51,562 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:12:51,562 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:12:51,563 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:12:51,563 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:12:51,800 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/jqla7lri-wandb-metadata.json +2025-01-06 17:12:51,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:12:51,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:12:51,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:12:51,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:12:51,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:51,814 INFO HandlerThread:2086753 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/tb" + save: true +} +control { + relay_id: "83c02b91baf3499fb941f0c7d9259a5b" +} +_info { + stream_id: "fnq3cpdm" +} + +2025-01-06 17:12:51,814 DEBUG HandlerThread:2086753 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:12:51,843 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/1qaw1h0w-diff.patch +2025-01-06 17:12:51,862 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/rq6hd07z-code/FISH/eval_robot.py +2025-01-06 17:12:51,947 DEBUG SenderThread:2086753 [sender.py:send():379] send: telemetry +2025-01-06 17:12:51,947 DEBUG SenderThread:2086753 [sender.py:send():379] send: tbrecord +2025-01-06 17:12:51,948 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:12:51,948 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 with policy live +2025-01-06 17:12:52,131 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/a0bptdxd-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch +2025-01-06 17:12:52,373 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code +2025-01-06 17:12:52,374 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH +2025-01-06 17:12:52,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:53,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:54,372 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:12:54,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:55,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:56,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:56,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:57,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:58,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:59,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:00,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:01,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:02,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:02,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:03,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:04,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:05,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:06,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:06,811 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:07,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:07,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:08,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:09,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:10,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:11,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:12,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:13,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:13,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:14,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:15,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:16,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:17,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:18,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:18,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:19,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:20,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:21,811 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:21,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:21,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:22,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:23,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:23,890 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:24,380 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:13:24,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:25,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:26,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:27,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:28,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:29,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:29,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:30,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:31,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:32,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:33,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:34,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:34,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:35,087 DEBUG SenderThread:2086753 [sender.py:send():379] send: config +2025-01-06 17:13:35,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:36,384 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:36,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:36,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:37,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:38,384 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:38,845 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:39,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:39,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:40,385 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:40,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:41,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:42,385 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:42,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:43,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:44,386 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:44,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:45,569 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:45,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:46,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:47,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:48,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:49,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:50,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:50,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:51,471 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:13:51,472 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:13:51,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:13:51,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:13:51,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:52,388 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:52,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:53,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:54,389 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:54,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:55,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:56,389 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:13:56,445 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:13:56,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:57,390 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:13:57,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:58,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:13:59,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:00,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:01,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:01,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:02,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:03,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:04,821 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:05,822 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:06,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:06,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:06,941 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:07,822 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:08,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:09,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:10,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:11,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:12,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:12,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:13,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:14,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:15,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:16,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:17,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:17,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:18,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:19,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:20,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:21,472 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:14:21,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:21,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:21,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:22,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:22,895 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:23,106 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:14:23,106 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 with policy now +2025-01-06 17:14:23,106 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:14:23,107 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:14:23,108 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:14:23,108 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:14:23,370 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/wdfckj65-media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,397 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos +2025-01-06 17:14:23,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media +2025-01-06 17:14:23,692 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/nrtcly84-media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:14:23,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:24,398 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:24,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:25,824 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:26,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:27,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:28,399 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:28,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:28,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:29,825 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:30,400 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:30,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:31,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:32,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:33,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:34,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:34,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:35,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:36,812 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:36,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:37,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:38,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:39,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:39,963 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:40,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:41,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:42,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:43,404 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:14:43,643 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/7pvsuix8-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:14:43,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:44,405 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:44,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:45,737 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:45,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:46,405 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:46,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:47,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:48,406 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:14:48,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:49,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:50,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:50,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:51,473 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:14:51,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:14:51,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:14:51,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:52,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:53,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:54,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:55,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:55,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:14:56,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:57,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:58,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:14:59,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:00,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:01,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:01,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:02,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:03,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:04,829 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:05,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:06,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:06,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:06,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:06,906 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:07,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:08,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:09,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:10,830 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:11,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:12,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:12,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:13,831 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:14,415 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:14,493 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:15:14,493 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 with policy now +2025-01-06 17:15:14,496 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:15:14,496 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:15:14,496 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:15:14,496 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:15:14,758 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/dozw7j6p-media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:14,834 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:15,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:15:15,733 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/47o71szn-media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:15:15,833 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:16,416 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:16,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:17,818 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:17,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:18,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:19,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:20,417 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:20,835 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:21,473 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:15:21,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:21,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:21,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:22,418 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:22,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:22,889 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:23,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:24,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:25,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:26,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:27,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:28,478 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:28,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:29,272 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:15:29,272 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:15:29,273 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:15:29,421 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:15:29,422 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:15:29,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:30,422 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:30,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:31,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:32,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:33,683 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:33,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:34,423 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:15:34,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:35,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:36,812 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:36,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:36,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:37,837 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:38,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:38,939 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:39,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:40,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:41,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:42,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:43,838 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:44,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:44,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:45,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:46,839 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:47,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:48,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:49,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:50,817 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:50,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:51,474 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:15:51,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:15:51,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:15:51,853 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:52,840 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:53,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:54,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:55,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:55,915 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:15:56,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:57,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:58,841 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:15:59,842 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:00,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:00,505 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:16:00,505 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 with policy now +2025-01-06 17:16:00,507 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:16:00,508 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:16:00,508 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:16:00,508 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:16:00,758 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/djjo0upu-media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:00,847 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:01,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:16:01,432 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:01,433 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:16:01,530 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:01,705 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/0hj6nayy-media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:16:01,846 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:02,433 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:02,857 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:03,855 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:04,859 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:05,860 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:06,434 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:06,739 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:06,813 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:06,813 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:06,856 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:07,862 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:08,435 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:08,865 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:09,862 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:10,859 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:11,826 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:11,860 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:12,863 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:13,861 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:14,863 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:15,866 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:16,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:17,820 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:17,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:18,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:19,873 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:20,869 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:21,474 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:16:21,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:21,817 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:21,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:22,867 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:22,948 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:23,868 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:24,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:25,870 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:26,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:27,872 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:28,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:28,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:29,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:30,871 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:31,874 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:32,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:33,877 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:34,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:34,879 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:35,878 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:36,823 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:36,823 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:36,878 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:37,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:38,444 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:16:38,689 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/9bxsd8t8-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:16:38,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:39,139 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:16:39,140 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:16:39,140 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:16:39,445 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:16:39,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:40,445 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:40,465 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:40,885 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:41,886 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:42,880 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:43,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:44,446 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:44,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:45,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:46,405 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:46,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:47,881 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:48,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:49,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:50,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:51,475 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:16:51,475 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:51,816 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:16:51,817 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:16:51,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:52,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:53,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:54,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:55,882 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:56,827 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:16:56,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:57,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:58,451 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:16:58,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:16:59,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:00,883 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:01,828 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:01,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:02,452 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:02,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:03,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:04,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:05,884 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:06,814 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:06,815 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:06,887 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:06,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:07,888 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:08,887 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:09,890 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:10,455 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:10,610 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:17:10,610 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 with policy now +2025-01-06 17:17:10,613 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:17:10,614 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:10,614 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:10,614 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:10,891 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/kf7o4uj8-media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:10,892 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:11,455 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:11,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:11,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:17:11,742 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/zif8y681-media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:17:11,893 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:12,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:12,836 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:12,894 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:13,456 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:13,894 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:14,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:15,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:16,457 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:16,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:17,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:17,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:18,458 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:18,896 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:19,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:20,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:21,475 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:17:21,819 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:21,819 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:21,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:22,901 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:23,192 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:23,193 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:23,193 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:23,193 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:23,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:17:23,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:23,903 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:24,460 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:24,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:25,904 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:26,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:27,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:28,461 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:28,898 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:29,034 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:29,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:30,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:31,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:32,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:33,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:34,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:34,899 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:35,908 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:36,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:36,815 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:36,909 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:37,909 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:38,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:39,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:39,971 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:40,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:41,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:42,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:43,910 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:44,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:45,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:45,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:46,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:47,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:48,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:49,914 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:50,832 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:50,911 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:51,476 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:17:51,815 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:17:51,816 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:17:51,912 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:52,917 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:53,918 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:54,470 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:54,923 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:55,177 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:17:55,177 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 with policy now +2025-01-06 17:17:55,180 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:17:55,181 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:17:55,181 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:17:55,181 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:17:55,431 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/1jvb2v6t-media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,470 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:17:55,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval +2025-01-06 17:17:55,735 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/adsnt5y9-media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:17:55,924 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:56,206 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:17:56,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:56,929 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:57,471 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:17:57,939 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:58,947 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:17:59,937 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:00,472 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:00,936 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:01,421 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:01,952 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:02,473 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:02,950 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:03,947 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:04,177 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:18:04,177 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:18:04,396 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table +2025-01-06 17:18:04,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval +2025-01-06 17:18:04,649 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/9wen6loj-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:04,683 INFO wandb-upload_1:2086753 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbuwzuhnv +2025-01-06 17:18:04,756 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:18:04,767 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:18:04,792 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/iicmhn41-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:04,942 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:05,091 INFO SenderThread:2086753 [sender.py:send_request_log_artifact():1525] logged artifact run-fnq3cpdm-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkxNDcxNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwOTY0MA==', 'latestArtifact': None}} +2025-01-06 17:18:05,091 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:18:05,091 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:18:05,091 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:05,092 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,093 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,093 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:05,094 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,097 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,098 DEBUG SenderThread:2086753 [sender.py:send():379] send: telemetry +2025-01-06 17:18:05,098 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,099 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,099 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,101 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,101 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,103 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,103 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,105 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,105 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,105 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,105 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,107 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,107 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,109 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,110 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,110 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,110 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,112 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,112 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,114 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,114 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:05,115 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:05,115 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:18:05,338 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/904i8vbd-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:05,471 INFO wandb-upload_0:2086753 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1n0gr8op +2025-01-06 17:18:05,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:05,474 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:05,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:05,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval +2025-01-06 17:18:05,942 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:06,020 INFO SenderThread:2086753 [sender.py:send_request_log_artifact():1525] logged artifact run-fnq3cpdm-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkxNDc0Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwOTY0Mg==', 'latestArtifact': None}} +2025-01-06 17:18:06,020 DEBUG SenderThread:2086753 [sender.py:send():379] send: files +2025-01-06 17:18:06,020 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json with policy now +2025-01-06 17:18:06,021 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:06,021 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:06,021 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:06,206 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/7p212ht7-media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:06,450 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/k8d8b30d-media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:06,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:06,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:06,491 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:06,537 DEBUG SenderThread:2086753 [sender.py:send():379] send: exit +2025-01-06 17:18:06,537 INFO SenderThread:2086753 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_exit():588] handling runtime: 315 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_exit():594] send defer +2025-01-06 17:18:06,538 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,538 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:18:06,538 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:18:06,538 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:18:06,538 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:18:06,539 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,539 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:18:06,539 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:18:06,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:18:06,539 INFO HandlerThread:2086753 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:18:06,539 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:18:06,539 DEBUG SystemMonitor:2086753 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:18:06,540 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:18:06,540 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:18:06,611 INFO HandlerThread:2086753 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:18:06,611 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,611 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:18:06,611 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:18:06,611 DEBUG SenderThread:2086753 [sender.py:send():379] send: stats +2025-01-06 17:18:06,612 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,612 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:18:06,612 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:06,612 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:18:06,612 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:18:06,612 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:06,612 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:18:07,475 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:08,476 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:13,248 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:13,248 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:13,249 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,250 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:18:13,250 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send():379] send: history +2025-01-06 17:18:13,250 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:18:13,250 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,250 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:18:13,251 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:13,251 DEBUG SenderThread:2086753 [sender.py:send():379] send: summary +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:18:13,255 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:18:13,255 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:18:13,255 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,255 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:18:13,256 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,256 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:18:13,259 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:18:13,411 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:18:13,411 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:13,411 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:18:13,411 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:13,411 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:18:13,478 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:13,478 INFO Thread-12 :2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:18:13,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:18:14,574 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:14,574 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:14,574 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:18:14,574 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:18:14,574 INFO SenderThread:2086753 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:18:14,577 INFO SenderThread:2086753 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:18:14,579 INFO SenderThread:2086753 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:18:14,596 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:18:14,596 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:14,596 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:18:14,597 DEBUG SenderThread:2086753 [sender.py:send():379] send: artifact +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'versionIndex': 422}}} +2025-01-06 17:18:14,795 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:18:14,795 INFO SenderThread:2086753 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml config.yaml +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json wandb-summary.json +2025-01-06 17:18:15,479 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt requirements.txt +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log output.log +2025-01-06 17:18:15,480 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/diff.patch diff.patch +2025-01-06 17:18:15,483 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:18:15,488 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 media/videos/eval/0_eval_0_dfa73dbea3eafd7c0a83.mp4 +2025-01-06 17:18:15,489 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 media/videos/eval/2_eval_2_acb7fe7e446267787ce7.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 media/videos/eval/4_eval_4_0e609f284fc921a805bf.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 media/videos/eval/1_eval_1_796f3c4bcdaca1535e57.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 media/videos/eval/3_eval_3_13f7da30cdf24122b19f.mp4 +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json media/table/eval/total_success_rate_ci_6_c6b6ac0bad860bd9e38c.table.json +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:18:15,490 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:18:15,491 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,491 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:18:15,491 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,491 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:18:15,491 INFO SenderThread:2086753 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:18:15,539 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:15,540 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:15,675 INFO wandb-upload_1:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/config.yaml +2025-01-06 17:18:15,677 INFO wandb-upload_2:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/wandb-summary.json +2025-01-06 17:18:15,690 INFO wandb-upload_4:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/output.log +2025-01-06 17:18:15,724 INFO wandb-upload_0:2086753 [upload_job.py:push():130] Uploaded file /tmp/tmplzvobhq4wandb/ncgpht3y-events.out.tfevents.1736201571.leonmkim-ROG-Strix-G15CS-G15CS.2086588.0 +2025-01-06 17:18:15,744 INFO wandb-upload_3:2086753 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/files/requirements.txt +2025-01-06 17:18:15,945 INFO Thread-11 (_thread_body):2086753 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:18:15,945 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,945 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:18:15,945 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,945 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:18:15,945 INFO SenderThread:2086753 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:18:15,946 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:18:15,946 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:15,946 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:18:15,947 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:15,947 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:18:15,947 INFO SenderThread:2086753 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:18:16,181 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:16,181 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:18:16,181 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:18:16,181 INFO SenderThread:2086753 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:18:16,181 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:18:16,181 INFO HandlerThread:2086753 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send():379] send: final +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send():379] send: footer +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: defer +2025-01-06 17:18:16,182 INFO SenderThread:2086753 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:18:16,182 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:16,182 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:18:16,183 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:18:16,183 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:18:16,184 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:18:16,185 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:18:16,234 DEBUG SenderThread:2086753 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:18:16,234 INFO MainThread:2086753 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:18:16,235 INFO MainThread:2086753 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:18:16,235 INFO MainThread:2086753 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:18:16,235 DEBUG HandlerThread:2086753 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:18:16,235 INFO HandlerThread:2086753 [handler.py:finish():882] shutting down handler +2025-01-06 17:18:17,183 INFO WriterThread:2086753 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [sender.py:finish():1615] shutting down sender +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:18:17,234 INFO SenderThread:2086753 [file_pusher.py:join():175] waiting for file pusher diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug.log b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e9797b08153ba5a3999513417dcbfd7c7a88e3a6 --- /dev/null +++ b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Configure stats pid to 2086588 +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:12:51,040 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/settings +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug.log +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/logs/debug-internal.log +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm'} +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():619] starting backend +2025-01-06 17:12:51,041 INFO MainThread:2086588 [wandb_init.py:init():623] setting up manager +2025-01-06 17:12:51,046 INFO MainThread:2086588 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:12:51,048 INFO MainThread:2086588 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:12:51,060 INFO MainThread:2086588 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:12:51,065 INFO MainThread:2086588 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:12:51,389 INFO MainThread:2086588 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:12:51,457 INFO MainThread:2086588 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:12:51,457 INFO MainThread:2086588 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:12:51,812 INFO MainThread:2086588 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:12:51,814 INFO MainThread:2086588 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:12:51,814 INFO MainThread:2086588 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/fnq3cpdm/tb, True +2025-01-06 17:13:35,086 INFO MainThread:2086588 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:18:17,236 WARNING MsgRouterThr:2086588 [router.py:message_loop():77] message_loop has been closed diff --git a/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..55e81864b3a94546863ad5786892f6bf0405b6a3 Binary files /dev/null and b/fnq3cpdm/wandb/run-20250106_171251-fnq3cpdm/run-fnq3cpdm.wandb differ diff --git a/hle5fmwc/episode_rosbags/aligned_depth_to_color_K.npy b/hle5fmwc/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hle5fmwc/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hle5fmwc/episode_rosbags/cam_tf_world.npy b/hle5fmwc/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/hle5fmwc/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/hle5fmwc/episode_rosbags/color_K.npy b/hle5fmwc/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/hle5fmwc/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/hle5fmwc/episode_rosbags/depth_K.npy b/hle5fmwc/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/hle5fmwc/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/hle5fmwc/eval_robot.log b/hle5fmwc/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ecce48a2e769a0f5ef1727c71f3ab4949f03b041 --- /dev/null +++ b/hle5fmwc/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:54:37,763][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:54:37,766][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/hle5fmwc/eval_video/0_eval.mp4 b/hle5fmwc/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a43bd2c760077ab3a9b6e16ee3e5ccae53520408 --- /dev/null +++ b/hle5fmwc/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:512fdb13a887694fdda3271d7e005e56d7c6a5f9836972a8ddb652d509b80ca5 +size 1129489 diff --git a/hle5fmwc/eval_video/1_eval.mp4 b/hle5fmwc/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d41ad37cbd37132c8f07db7987e6ad5e80e5c9cb --- /dev/null +++ b/hle5fmwc/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5a9fd56a60dca233f4596a84e3f7435cce734452f1d6c78aac0cac6cf68b215 +size 1135938 diff --git a/hle5fmwc/eval_video/2_eval.mp4 b/hle5fmwc/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85286330007eece40f56ecfb77f25ae8ab713296 --- /dev/null +++ b/hle5fmwc/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:482df1c0b75a61d8a84c5df4016c8c1c8753f3496a3397421372988827ce6468 +size 1138248 diff --git a/hle5fmwc/eval_video/3_eval.mp4 b/hle5fmwc/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4340f0d48ba370f2b2b4f9e54e7c0131d4ba4a4f --- /dev/null +++ b/hle5fmwc/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697753ebc9b72b44c2363c94e8a03612781b4783cb7c873af9c1f13022462b26 +size 1145544 diff --git a/hle5fmwc/eval_video/4_eval.mp4 b/hle5fmwc/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3e75f3b0326dfd8274bdfa07eec901ebbac8001 --- /dev/null +++ b/hle5fmwc/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26772b2cd069309bc708cba9cdcb54897f36d4a20f6315c9c9e687728c343941 +size 1143811 diff --git a/hle5fmwc/tb/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 b/hle5fmwc/tb/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 new file mode 100644 index 0000000000000000000000000000000000000000..a119c409df845190c860372230f0f31dd829f483 --- /dev/null +++ b/hle5fmwc/tb/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e2ac53ee79e92f21e1b605b63d588df3690e46950c8ec2a64a83f1b5462c417 +size 6539 diff --git a/hle5fmwc/wandb/debug-internal.log b/hle5fmwc/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4d5e25cfdaffcaf3320d502d2583c3a393bea395 --- /dev/null +++ b/hle5fmwc/wandb/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-06 20:54:43,562 INFO StreamThr :2116885 [internal.py:wandb_internal():85] W&B internal server running at pid: 2116885, started at: 2025-01-06 20:54:43.562013 +2025-01-06 20:54:43,563 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:54:43,580 INFO WriterThread:2116885 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb +2025-01-06 20:54:43,580 DEBUG SenderThread:2116885 [sender.py:send():379] send: header +2025-01-06 20:54:43,587 DEBUG SenderThread:2116885 [sender.py:send():379] send: run +2025-01-06 20:54:43,882 INFO SenderThread:2116885 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files +2025-01-06 20:54:43,882 INFO SenderThread:2116885 [sender.py:_start_run_threads():1188] run started: hle5fmwc with start time 1736214883.562129 +2025-01-06 20:54:43,898 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:54:43,898 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:54:43,970 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:54:43,975 DEBUG HandlerThread:2116885 [system_info.py:__init__():26] System info init +2025-01-06 20:54:43,975 DEBUG HandlerThread:2116885 [system_info.py:__init__():41] System info init done +2025-01-06 20:54:43,975 INFO HandlerThread:2116885 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:54:43,975 INFO SystemMonitor:2116885 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:54:43,975 INFO HandlerThread:2116885 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:54:43,975 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:54:43,976 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:54:43,977 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:54:43,977 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:54:43,978 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:54:43,987 DEBUG HandlerThread:2116885 [system_info.py:probe():152] Probing system +2025-01-06 20:54:43,988 DEBUG HandlerThread:2116885 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:probe():200] Probing system done +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:54:43.987359', 'startedAt': '2025-01-07T01:54:43.554646', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3746.592375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.493, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.072, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 531.8875846862793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:54:43,993 INFO HandlerThread:2116885 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:54:43,993 INFO HandlerThread:2116885 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:_save_code():44] Saving code +2025-01-06 20:54:43,995 DEBUG HandlerThread:2116885 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:54:43,995 DEBUG HandlerThread:2116885 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:54:44,062 DEBUG HandlerThread:2116885 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:54:44,063 INFO HandlerThread:2116885 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:54:44,071 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:54:44,071 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:54:44,072 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:54:44,072 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:54:44,333 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/recile0m-wandb-metadata.json +2025-01-06 20:54:44,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:54:44,334 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:54:44,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:54:44,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:54:44,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:44,349 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/6v0a63hy-diff.patch +2025-01-06 20:54:44,378 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/m1z118xw-code/FISH/eval_robot.py +2025-01-06 20:54:44,381 INFO HandlerThread:2116885 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/tb" + save: true +} +control { + relay_id: "98dbe3f98f56404d92509137384227db" +} +_info { + stream_id: "hle5fmwc" +} + +2025-01-06 20:54:44,381 DEBUG HandlerThread:2116885 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:54:44,481 DEBUG SenderThread:2116885 [sender.py:send():379] send: telemetry +2025-01-06 20:54:44,481 DEBUG SenderThread:2116885 [sender.py:send():379] send: tbrecord +2025-01-06 20:54:44,482 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:54:44,482 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 with policy live +2025-01-06 20:54:44,669 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/fhtketxu-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:54:44,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch +2025-01-06 20:54:44,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH +2025-01-06 20:54:45,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:46,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:46,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:54:47,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:48,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:49,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:54:49,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:50,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:51,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:52,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:53,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:54,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:54:54,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:55,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:56,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:57,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:58,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:59,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:54:59,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:54:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:59,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:00,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:01,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:01,443 DEBUG SenderThread:2116885 [sender.py:send():379] send: config +2025-01-06 20:55:02,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:02,887 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:03,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:04,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:04,888 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:05,162 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:05,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:06,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:06,888 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:07,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:08,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:08,889 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:09,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:10,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:10,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:10,889 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:11,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:12,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:13,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:14,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:14,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:14,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:15,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:15,408 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:15,891 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:55:16,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:17,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:18,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:19,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:20,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:20,561 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:20,892 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:21,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:22,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:23,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:24,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:24,893 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:25,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:26,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:26,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:27,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:28,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:29,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:29,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:29,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:30,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:31,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:31,479 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:32,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:33,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:34,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:35,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:36,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:37,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:37,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:38,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:39,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:40,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:40,897 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:41,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:42,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:43,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:43,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:43,978 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:55:43,978 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:55:44,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:44,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:44,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:45,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:46,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:47,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:48,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:48,452 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:49,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:50,219 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:55:50,219 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 with policy now +2025-01-06 20:55:50,219 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:55:50,220 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:55:50,220 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:55:50,220 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:55:50,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:50,474 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/6ilu7nrw-media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media +2025-01-06 20:55:51,191 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/cdx0kmfh-media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:51,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:52,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:52,901 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:53,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:53,934 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:54,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:54,901 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:55,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:56,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:57,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:58,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:58,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:59,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:59,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:59,453 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:59,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:00,142 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/55uf3lje-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:00,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:00,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:01,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:02,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:02,904 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:03,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:04,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:04,905 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:05,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:05,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:06,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:07,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:08,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:09,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:10,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:10,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:11,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:12,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:13,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:13,979 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:56:14,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:14,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:14,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:15,347 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:15,429 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:16,347 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:17,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:18,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:19,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:20,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:21,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:21,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:22,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:23,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:24,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:25,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:26,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:26,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:27,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:28,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:29,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:29,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:29,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:30,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:31,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:31,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:31,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:56:31,420 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 with policy now +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:56:31,421 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:56:31,668 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/tx0x10a6-media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:56:32,193 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/2r95y80y-media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:32,353 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:32,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:33,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:34,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:35,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:36,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:36,915 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:37,205 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:37,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:38,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:38,915 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:39,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:40,365 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:40,815 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:56:40,816 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:56:40,816 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:56:40,916 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:56:40,916 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:41,389 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:42,361 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:42,390 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:42,917 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:43,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:43,979 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:56:44,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:44,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:44,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:44,918 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:45,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:46,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:46,918 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:47,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:47,459 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:48,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:49,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:50,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:51,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:52,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:53,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:53,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:54,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:55,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:56,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:57,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:58,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:58,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:59,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:59,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:00,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:01,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:02,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:03,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:03,457 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:04,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:05,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:06,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:07,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:08,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:09,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:09,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:10,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:11,360 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:12,232 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:57:12,232 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 with policy now +2025-01-06 20:57:12,236 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:57:12,236 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:12,236 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:12,237 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:12,365 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:12,481 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/4yyjusr1-media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:57:13,227 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/qpp18kiv-media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:13,368 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:13,980 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:57:14,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:14,342 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:14,389 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:14,412 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:14,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:15,368 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:16,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:16,928 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:17,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:18,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:19,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:19,970 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:20,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:20,640 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:20,640 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:20,641 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:20,929 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:20,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:57:20,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:21,165 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/cctcmkhu-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:57:21,374 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:22,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:22,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:23,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:24,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:24,931 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:25,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:25,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:26,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:26,931 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:27,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:28,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:28,932 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:29,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:29,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:29,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:30,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:30,483 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:31,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:32,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:33,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:34,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:35,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:36,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:36,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:37,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:38,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:39,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:40,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:41,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:42,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:42,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:43,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:43,980 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:57:44,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:44,337 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:44,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:45,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:46,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:47,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:47,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:48,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:49,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:50,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:51,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:52,143 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:57:52,143 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 with policy now +2025-01-06 20:57:52,150 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:57:52,150 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:52,150 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:52,151 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:52,379 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:52,395 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/35co50qv-media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:57:53,166 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:53,242 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/vq8kfyfq-media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:53,387 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:54,385 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:55,392 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:56,395 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:56,941 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:57,395 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:58,403 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:58,760 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:59,353 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:59,354 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:59,397 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:00,396 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:00,943 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:01,399 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:02,399 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:03,400 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:04,361 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:04,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:05,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:06,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:07,407 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:08,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:09,407 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:10,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:10,405 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:11,410 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:12,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:13,414 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:13,981 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:58:14,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:14,345 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:14,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:15,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:15,489 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:16,412 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:16,708 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:16,708 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:16,708 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:16,948 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:16,948 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:17,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:18,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:18,949 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:19,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:20,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:20,949 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:21,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:21,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:22,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:22,950 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:23,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:24,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:25,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:26,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:27,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:27,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:28,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:29,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:29,340 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:29,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:30,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:31,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:32,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:32,702 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:32,953 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:33,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:34,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:34,954 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:35,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:36,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:37,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:37,706 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:38,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:38,956 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:39,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:40,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:41,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:42,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:43,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:43,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:43,981 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:58:44,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:44,340 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:44,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:44,958 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:45,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:46,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:47,425 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:48,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:48,445 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:48,636 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:48,636 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 with policy now +2025-01-06 20:58:48,640 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:48,640 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:48,640 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:48,640 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:48,878 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/qyd8pmgm-media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:48,959 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:48,959 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:48,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:48,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:58:49,275 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/9auudd0t-media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:49,425 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:50,438 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:50,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:51,433 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:52,435 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:53,436 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:53,517 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:54,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:54,961 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:55,432 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:56,438 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:56,962 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:57,449 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:57,754 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:57,754 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:57,755 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:57,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:57,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,177 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:58:58,178 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:58:58,208 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/7yp6xd7m-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,370 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:58,436 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:58,675 INFO wandb-upload_0:2116885 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgwipaqn0 +2025-01-06 20:58:58,763 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:58:58,777 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table +2025-01-06 20:58:59,060 INFO SenderThread:2116885 [sender.py:send_request_log_artifact():1525] logged artifact run-hle5fmwc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM1NDE0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1ODg3OQ==', 'latestArtifact': None}} +2025-01-06 20:58:59,060 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:59,060 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:58:59,060 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:59,060 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:59,061 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,061 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,061 DEBUG SenderThread:2116885 [sender.py:send():379] send: telemetry +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,062 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,062 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,063 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,063 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,063 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,063 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,064 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,064 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,065 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,065 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,065 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,065 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,066 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,066 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:58:59,239 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/acbwd561-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:59,260 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/2l6wdk20-media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:58:59,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:59,422 INFO wandb-upload_0:2116885 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm6b7sljq +2025-01-06 20:58:59,437 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:59,478 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/fwkuobur-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:59,770 INFO SenderThread:2116885 [sender.py:send_request_log_artifact():1525] logged artifact run-hle5fmwc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM1NDE2MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1ODg4NA==', 'latestArtifact': None}} +2025-01-06 20:58:59,770 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:59,770 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json with policy now +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,771 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:59,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:00,032 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/gs46ew22-media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:59:00,105 DEBUG SenderThread:2116885 [sender.py:send():379] send: exit +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():588] handling runtime: 256 +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():594] send defer +2025-01-06 20:59:00,106 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,106 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:59:00,106 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:59:00,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:59:00,107 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:59:00,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:59:00,107 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:59:00,107 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:59:00,108 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:59:00,108 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:59:00,160 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,160 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:59:00,160 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:59:00,161 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:59:00,161 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,161 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:59:00,161 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,161 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:59:00,161 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:59:00,161 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,161 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:59:00,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:00,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:04,966 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:59:06,761 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:06,762 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:59:06,762 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:59:06,762 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,763 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:59:06,763 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:59:06,764 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,765 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:59:06,765 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:06,765 DEBUG SenderThread:2116885 [sender.py:send():379] send: summary +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:06,766 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,766 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:59:06,766 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,766 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:59:06,767 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,767 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:59:06,966 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:07,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:08,465 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:59:08,465 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:08,465 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,465 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:59:08,466 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,466 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:59:08,466 INFO SenderThread:2116885 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:59:08,468 INFO SenderThread:2116885 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:59:08,470 INFO SenderThread:2116885 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:59:08,484 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:59:08,485 DEBUG SenderThread:2116885 [sender.py:send():379] send: artifact +2025-01-06 20:59:08,488 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,488 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:59:08,690 INFO SenderThread:2116885 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:59:08,691 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,691 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:59:08,691 INFO SenderThread:2116885 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml config.yaml +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json wandb-summary.json +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt requirements.txt +2025-01-06 20:59:08,970 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:59:08,970 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log output.log +2025-01-06 20:59:08,972 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch diff.patch +2025-01-06 20:59:08,973 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:59:08,977 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:59:08,978 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,978 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:59:08,979 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,979 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:59:08,979 INFO SenderThread:2116885 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:59:09,108 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,108 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,148 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:59:09,163 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:09,210 INFO wandb-upload_3:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt +2025-01-06 20:59:09,236 INFO wandb-upload_4:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:09,247 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/hzghrb2g-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:59:09,448 INFO Thread-11 (_thread_body):2116885 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:59:09,448 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,448 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:59:09,448 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,448 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:59:09,448 INFO SenderThread:2116885 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:59:09,450 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,450 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:59:09,450 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:59:09,648 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,648 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:59:09,648 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:59:09,648 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,649 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send():379] send: final +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send():379] send: footer +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,649 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:59:09,649 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,650 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:59:09,651 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:59:09,652 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:59:09,653 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:59:09,712 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,712 INFO MainThread:2116885 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:59:09,713 INFO MainThread:2116885 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:59:09,713 INFO MainThread:2116885 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:59:09,713 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:59:09,713 INFO HandlerThread:2116885 [handler.py:finish():882] shutting down handler +2025-01-06 20:59:10,651 INFO WriterThread:2116885 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [sender.py:finish():1615] shutting down sender +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [file_pusher.py:join():175] waiting for file pusher diff --git a/hle5fmwc/wandb/debug.log b/hle5fmwc/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e5f100751909aab93f92a101c2fe2bd6b56e768a --- /dev/null +++ b/hle5fmwc/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Configure stats pid to 2116716 +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/settings +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug.log +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug-internal.log +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc'} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():619] starting backend +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():623] setting up manager +2025-01-06 20:54:43,561 INFO MainThread:2116716 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:54:43,561 INFO MainThread:2116716 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:54:43,579 INFO MainThread:2116716 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:54:43,585 INFO MainThread:2116716 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:54:43,898 INFO MainThread:2116716 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:54:43,964 INFO MainThread:2116716 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:54:43,965 INFO MainThread:2116716 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:54:44,334 INFO MainThread:2116716 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:54:44,337 INFO MainThread:2116716 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:54:44,338 INFO MainThread:2116716 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/tb, True +2025-01-06 20:55:01,442 INFO MainThread:2116716 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:59:10,714 WARNING MsgRouterThr:2116716 [router.py:message_loop():77] message_loop has been closed diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c803ebd4dc834c9d525ae39056bd7de1bb637cee --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736214883 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d23be7987bea3201fc724ff3a6347a1c58944871 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[16, 25, 0.64, 0.4000362456044806, 0.8355595766826898, 0.44931596518576544, 0.8190537839489838, 0.4797478020231696, 0.7782795726696401, 0.4988887012007075, 0.7662575095359437]]} \ No newline at end of file diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a43bd2c760077ab3a9b6e16ee3e5ccae53520408 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:512fdb13a887694fdda3271d7e005e56d7c6a5f9836972a8ddb652d509b80ca5 +size 1129489 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d41ad37cbd37132c8f07db7987e6ad5e80e5c9cb --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5a9fd56a60dca233f4596a84e3f7435cce734452f1d6c78aac0cac6cf68b215 +size 1135938 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85286330007eece40f56ecfb77f25ae8ab713296 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:482df1c0b75a61d8a84c5df4016c8c1c8753f3496a3397421372988827ce6468 +size 1138248 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4340f0d48ba370f2b2b4f9e54e7c0131d4ba4a4f --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:697753ebc9b72b44c2363c94e8a03612781b4783cb7c873af9c1f13022462b26 +size 1145544 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b3e75f3b0326dfd8274bdfa07eec901ebbac8001 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26772b2cd069309bc708cba9cdcb54897f36d4a20f6315c9c9e687728c343941 +size 1143811 diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..25fb2502e66a3f5cbeb62b67e7f073e84300cbd0 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736214901.443528]: resetting environment +[INFO] [1736214901.448575]: cleared current plan +[INFO] [1736214901.449420]: moving to home +[INFO] [1736214902.451688]: reached home +[INFO] [1736214902.452971]: reset action history +[INFO] [1736214905.161618]: environment reset +Starting episode 0 +[INFO] [1736214905.163876]: resetting environment +[INFO] [1736214905.166556]: cleared current plan +[INFO] [1736214905.167441]: moving to home +[INFO] [1736214906.169647]: reached home +[INFO] [1736214906.171681]: reset action history +[INFO] [1736214908.878341]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736214918.514549]: resetting environment +[INFO] [1736214918.517491]: cleared current plan +[INFO] [1736214918.518508]: moving to home +[INFO] [1736214919.520810]: reached home +[INFO] [1736214919.522187]: reset action history +[INFO] [1736214922.228764]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/episode_rosbags/episode_0_2025-01-06-20-55-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2116716.log +started roslaunch server http://158.130.50.37:34253/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2116985] +process[print_text-2]: started with pid [2117009] +process[rosbag_record-3]: started with pid [2117010] +started bagging! +For topic gripper_width: timestamp difference is 104551088 for nearest: 1736214938666555177 - target: 1736214938562004089 at idx 206 +[INFO] [1736214949.255070]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1736214950.009294]: Stored episode 1. +[INFO] [1736214950.010047]: Saving video... +[INFO] [1736214950.206921]: Video saved! +Starting episode 1 +[INFO] [1736214950.221920]: resetting environment +[INFO] [1736214950.231065]: cleared current plan +[INFO] [1736214950.231352]: moving to home +[INFO] [1736214953.934000]: reached home +[INFO] [1736214953.934898]: reset action history +[INFO] [1736214956.643337]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736214959.477928]: resetting environment +[INFO] [1736214959.480361]: cleared current plan +[INFO] [1736214959.480527]: moving to home +[INFO] [1736214960.481389]: reached home +[INFO] [1736214960.481909]: reset action history +[INFO] [1736214963.188617]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/episode_rosbags/episode_1_2025-01-06-20-56-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2116716.log +started roslaunch server http://158.130.50.37:34393/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2117120] +process[print_text-5]: started with pid [2117121] +process[rosbag_record-6]: started with pid [2117122] +started bagging! +[INFO] [1736214990.153530]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736214990.910739]: Stored episode 2. +[INFO] [1736214990.911583]: Saving video... +[INFO] [1736214991.374445]: Video saved! +Starting episode 2 +[INFO] [1736214991.423111]: resetting environment +[INFO] [1736214991.447916]: cleared current plan +[INFO] [1736214991.451589]: moving to home +[INFO] [1736214995.160362]: reached home +[INFO] [1736214995.165464]: reset action history +[INFO] [1736214997.938209]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215000.335220]: resetting environment +[INFO] [1736215000.344071]: cleared current plan +[INFO] [1736215000.345422]: moving to home +[INFO] [1736215001.347691]: reached home +[INFO] [1736215001.348491]: reset action history +[INFO] [1736215004.105115]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/episode_rosbags/episode_2_2025-01-06-20-56-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2116716.log +started roslaunch server http://158.130.50.37:43089/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2117255] +process[print_text-8]: started with pid [2117256] +process[rosbag_record-9]: started with pid [2117280] +started bagging! +[INFO] [1736215031.101961]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736215031.891068]: Stored episode 3. +[INFO] [1736215031.896875]: Saving video... +[INFO] [1736215032.182525]: Video saved! +Starting episode 3 +[INFO] [1736215032.238862]: resetting environment +[INFO] [1736215032.250433]: cleared current plan +[INFO] [1736215032.252513]: moving to home +[INFO] [1736215036.066855]: reached home +[INFO] [1736215036.069214]: reset action history +[INFO] [1736215038.854243]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215040.001039]: resetting environment +[INFO] [1736215040.008011]: cleared current plan +[INFO] [1736215040.008206]: moving to home +[INFO] [1736215041.009296]: reached home +[INFO] [1736215041.014201]: reset action history +[INFO] [1736215043.797424]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/episode_rosbags/episode_3_2025-01-06-20-57-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2116716.log +started roslaunch server http://158.130.50.37:40137/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2117412] +process[print_text-11]: started with pid [2117413] +process[rosbag_record-12]: started with pid [2117414] +started bagging! +For topic gripper_width: timestamp difference is -108089867 for nearest: 1736215046633220729 - target: 1736215046741310596 at idx 42 +[INFO] [1736215071.040994]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736215071.790760]: Stored episode 4. +[INFO] [1736215071.801417]: Saving video... +[INFO] [1736215072.100465]: Video saved! +Starting episode 4 +[INFO] [1736215072.152575]: resetting environment +[INFO] [1736215072.163470]: cleared current plan +[INFO] [1736215072.165716]: moving to home +[INFO] [1736215075.977439]: reached home +[INFO] [1736215075.979032]: reset action history +[INFO] [1736215078.759661]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215096.256585]: resetting environment +[INFO] [1736215096.277787]: cleared current plan +[INFO] [1736215096.281384]: moving to home +[INFO] [1736215097.289007]: reached home +[INFO] [1736215097.291251]: reset action history +[INFO] [1736215100.076237]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/episode_rosbags/episode_4_2025-01-06-20-58-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2116716.log +started roslaunch server http://158.130.50.37:36027/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2117525] +process[print_text-14]: started with pid [2117549] +process[rosbag_record-15]: started with pid [2117550] +started bagging! +For topic gripper_width: timestamp difference is 111151979 for nearest: 1736215111899885700 - target: 1736215111788733721 at idx 170 +For topic gripper_width: timestamp difference is 111781730 for nearest: 1736215112699884309 - target: 1736215112588102579 at idx 181 +For topic gripper_width: timestamp difference is 115855404 for nearest: 1736215117699911543 - target: 1736215117584056139 at idx 233 +For topic gripper_width: timestamp difference is 116011399 for nearest: 1736215117899914998 - target: 1736215117783903599 at idx 235 +For topic gripper_width: timestamp difference is 120631131 for nearest: 1736215123699886235 - target: 1736215123579255104 at idx 249 +[INFO] [1736215127.076591]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit[rosbag_record-15] killing on exit +[INFO] [1736215127.958675]: Stored episode 5. +[INFO] [1736215127.959519]: Saving video... +[WARN] [1736215128.444586]: Plan exhausted +[WARN] [1736215128.493583]: Plan exhausted +[WARN] [1736215128.525379]: Plan exhausted +[WARN] [1736215128.564665]: Plan exhausted +[WARN] [1736215128.606930]: Plan exhausted +[WARN] [1736215128.648771]: Plan exhausted +[INFO] [1736215128.592876]: Video saved! +[INFO] [1736215128.643821]: resetting environment +[INFO] [1736215128.657733]: cleared current plan +[INFO] [1736215128.664778]: moving to home +[INFO] [1736215132.474229]: reached home +[INFO] [1736215132.481471]: reset action history +[INFO] [1736215135.456604]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4732402a7bfed125ef3f2e1cd9eae0bdb5cfe6a0 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:54:43.987359", + "startedAt": "2025-01-07T01:54:43.554646", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3746.592375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4493.493, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.072, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 531.8875846862793 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e3ac904d61c233e7d1987879ad5841aaa06b13b4 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "512fdb13a887694fdda3271d7e005e56d7c6a5f9836972a8ddb652d509b80ca5", "size": 1129489, "path": "media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4"}, "global_step": 664, "_timestamp": 1736215138.6785018, "_runtime": 255.1163728237152, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "f5a9fd56a60dca233f4596a84e3f7435cce734452f1d6c78aac0cac6cf68b215", "size": 1135938, "path": "media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4"}, "eval/num_success": 5.0, "episode": 25.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "482df1c0b75a61d8a84c5df4016c8c1c8753f3496a3397421372988827ce6468", "size": 1138248, "path": "media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "697753ebc9b72b44c2363c94e8a03612781b4783cb7c873af9c1f13022462b26", "size": 1145544, "path": "media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "26772b2cd069309bc708cba9cdcb54897f36d4a20f6315c9c9e687728c343941", "size": 1143811, "path": "media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://k2h71hbpwuwvagp6bm4z6zg3u21ni60rkqy87o81dt9fesnit450cqjh8vj3o0ikaxoyre3f14ad5ueb0pf8swireaajlpvget37tf23suwbo7y0usp1mn9k747vqqv4/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://nvf04gyuyfbuhg0b7xul7x7w3b6r8a4culxp27alw9b3hb0ofjq2owuh24plilxaexflhtkw64qaef4eq03l2vimoezredia5o6ivb7q6i3n17ul9d788ppyb9ntgs5e:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 16, "total_num_episodes": 25, "total_success_rate": 0.64, "total_success_rate_lb_0.01": 0.4000362456044806, "total_success_rate_ub_0.01": 0.8355595766826898, "total_success_rate_lb_0.025": 0.44931596518576544, "total_success_rate_ub_0.025": 0.8190537839489838, "total_success_rate_lb_0.05": 0.4797478020231696, "total_success_rate_ub_0.05": 0.7782795726696401, "total_success_rate_lb_0.1": 0.4988887012007075, "total_success_rate_ub_0.1": 0.7662575095359437, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "fc242770b6dc78d2cc1752a3ddd237bb651e92677dcf1dcbfda1c89eb86a35c2", "size": 507, "artifact_path": "wandb-client-artifact://cs0brzs3z0mbqg1zyonjjxqo481fq5o3013awleqs79nmh0ta59lfvfa1h0r6vyzgjtrm9fo53seru0o99ds7zamg0a18cdfe8tpqu5k7qifj1zqupfd8px6rttlv0ca/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://l1wf104t3dvalmv6g5rmtv7ek4a3a9ovn8uho3f083ro7gly2gfbn18q86qpoqphtssvig7nm2j0p02v2i2fkysp51n5y6mynnck9rcxxn8xg101n0r75414vxjqz76e:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 256}, "eval/total_success_rate": 0.6399999856948853, "eval/total_success_rate_lb_0.01": 0.4000362455844879, "eval/total_success_rate_ub_0.01": 0.8355595469474792, "eval/total_success_rate_lb_0.025": 0.44931596517562866, "eval/total_success_rate_ub_0.025": 0.8190537691116333, "eval/total_success_rate_lb_0.05": 0.47974780201911926, "eval/total_success_rate_ub_0.05": 0.7782795429229736, "eval/total_success_rate_lb_0.1": 0.49888870120048523, "eval/total_success_rate_ub_0.1": 0.7662575244903564} \ No newline at end of file diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug-internal.log b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4d5e25cfdaffcaf3320d502d2583c3a393bea395 --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug-internal.log @@ -0,0 +1,753 @@ +2025-01-06 20:54:43,562 INFO StreamThr :2116885 [internal.py:wandb_internal():85] W&B internal server running at pid: 2116885, started at: 2025-01-06 20:54:43.562013 +2025-01-06 20:54:43,563 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:54:43,580 INFO WriterThread:2116885 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb +2025-01-06 20:54:43,580 DEBUG SenderThread:2116885 [sender.py:send():379] send: header +2025-01-06 20:54:43,587 DEBUG SenderThread:2116885 [sender.py:send():379] send: run +2025-01-06 20:54:43,882 INFO SenderThread:2116885 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files +2025-01-06 20:54:43,882 INFO SenderThread:2116885 [sender.py:_start_run_threads():1188] run started: hle5fmwc with start time 1736214883.562129 +2025-01-06 20:54:43,898 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:54:43,898 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:54:43,970 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:54:43,975 DEBUG HandlerThread:2116885 [system_info.py:__init__():26] System info init +2025-01-06 20:54:43,975 DEBUG HandlerThread:2116885 [system_info.py:__init__():41] System info init done +2025-01-06 20:54:43,975 INFO HandlerThread:2116885 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:54:43,975 INFO SystemMonitor:2116885 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:54:43,975 INFO HandlerThread:2116885 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:54:43,975 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:54:43,976 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:54:43,977 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:54:43,977 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:54:43,978 INFO SystemMonitor:2116885 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:54:43,987 DEBUG HandlerThread:2116885 [system_info.py:probe():152] Probing system +2025-01-06 20:54:43,988 DEBUG HandlerThread:2116885 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:probe():200] Probing system done +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:54:43.987359', 'startedAt': '2025-01-07T01:54:43.554646', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3746.592375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.493, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.072, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 531.8875846862793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:54:43,993 INFO HandlerThread:2116885 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:54:43,993 INFO HandlerThread:2116885 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:54:43,993 DEBUG HandlerThread:2116885 [system_info.py:_save_code():44] Saving code +2025-01-06 20:54:43,995 DEBUG HandlerThread:2116885 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:54:43,995 DEBUG HandlerThread:2116885 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:54:44,062 DEBUG HandlerThread:2116885 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:54:44,063 INFO HandlerThread:2116885 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:54:44,071 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:54:44,071 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:54:44,072 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:54:44,072 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:54:44,333 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/recile0m-wandb-metadata.json +2025-01-06 20:54:44,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:54:44,334 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:54:44,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:54:44,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:54:44,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:44,349 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/6v0a63hy-diff.patch +2025-01-06 20:54:44,378 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/m1z118xw-code/FISH/eval_robot.py +2025-01-06 20:54:44,381 INFO HandlerThread:2116885 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/tb" + save: true +} +control { + relay_id: "98dbe3f98f56404d92509137384227db" +} +_info { + stream_id: "hle5fmwc" +} + +2025-01-06 20:54:44,381 DEBUG HandlerThread:2116885 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:54:44,481 DEBUG SenderThread:2116885 [sender.py:send():379] send: telemetry +2025-01-06 20:54:44,481 DEBUG SenderThread:2116885 [sender.py:send():379] send: tbrecord +2025-01-06 20:54:44,482 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:54:44,482 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 with policy live +2025-01-06 20:54:44,669 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/fhtketxu-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:54:44,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch +2025-01-06 20:54:44,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code +2025-01-06 20:54:44,884 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH +2025-01-06 20:54:45,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:46,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:46,883 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:54:47,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:48,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:49,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:54:49,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:50,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:51,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:52,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:53,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:54,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:54:54,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:55,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:56,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:57,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:58,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:59,334 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:54:59,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:54:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:54:59,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:00,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:01,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:01,443 DEBUG SenderThread:2116885 [sender.py:send():379] send: config +2025-01-06 20:55:02,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:02,887 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:03,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:04,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:04,888 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:05,162 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:05,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:06,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:06,888 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:07,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:08,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:08,889 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:09,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:10,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:10,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:10,889 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:11,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:12,338 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:13,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:14,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:14,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:14,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:15,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:15,408 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:15,891 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:55:16,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:17,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:18,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:19,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:20,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:20,561 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:20,892 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:21,339 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:22,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:23,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:24,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:24,893 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:25,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:26,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:26,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:27,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:28,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:29,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:29,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:29,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:30,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:31,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:31,479 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:32,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:33,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:34,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:35,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:36,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:37,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:37,341 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:38,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:39,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:40,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:40,897 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:41,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:42,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:43,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:43,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:43,978 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:55:43,978 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:55:44,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:44,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:44,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:45,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:46,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:47,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:48,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:48,452 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:49,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:50,219 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:55:50,219 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 with policy now +2025-01-06 20:55:50,219 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:55:50,220 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:55:50,220 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:55:50,220 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:55:50,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:50,474 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/6ilu7nrw-media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos +2025-01-06 20:55:50,900 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media +2025-01-06 20:55:51,191 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/cdx0kmfh-media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:55:51,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:52,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:52,901 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:53,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:53,934 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:54,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:54,901 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:55,343 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:56,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:57,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:58,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:58,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:55:59,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:55:59,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:55:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:55:59,453 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:55:59,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:00,142 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/55uf3lje-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:00,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:00,903 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:01,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:02,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:02,904 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:03,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:04,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:04,905 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:05,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:05,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:06,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:07,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:08,345 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:09,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:10,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:10,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:11,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:12,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:13,346 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:13,979 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:56:14,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:14,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:14,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:15,347 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:15,429 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:16,347 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:17,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:18,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:19,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:20,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:21,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:21,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:22,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:23,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:24,348 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:25,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:26,337 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:26,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:27,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:28,349 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:29,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:29,335 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:29,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:30,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:31,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:31,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:31,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:56:31,420 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 with policy now +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:56:31,420 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:56:31,421 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:56:31,668 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/tx0x10a6-media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:31,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:56:32,193 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/2r95y80y-media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:56:32,353 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:32,913 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:33,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:34,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:35,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:36,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:36,915 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:37,205 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:37,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:38,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:38,915 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:39,354 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:40,365 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:40,815 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:56:40,816 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:56:40,816 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:56:40,916 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:56:40,916 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:56:41,389 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:42,361 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:42,390 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:42,917 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:43,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:43,979 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:56:44,335 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:44,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:44,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:44,918 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:45,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:46,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:46,918 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:56:47,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:47,459 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:48,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:49,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:50,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:51,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:52,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:53,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:53,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:54,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:55,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:56,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:57,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:58,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:56:58,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:56:59,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:56:59,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:56:59,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:00,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:01,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:02,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:03,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:03,457 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:04,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:05,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:06,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:07,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:08,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:09,350 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:09,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:10,359 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:11,360 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:12,232 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:57:12,232 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 with policy now +2025-01-06 20:57:12,236 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:57:12,236 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:12,236 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:12,237 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:12,365 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:12,481 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/4yyjusr1-media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:12,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:57:13,227 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/qpp18kiv-media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:57:13,368 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:13,980 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:57:14,342 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:14,342 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:14,389 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:14,412 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:14,927 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:15,368 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:16,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:16,928 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:17,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:18,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:19,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:19,970 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:20,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:20,640 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:20,640 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:20,641 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:20,929 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:20,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:57:20,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:21,165 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/cctcmkhu-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:57:21,374 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:22,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:22,930 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:23,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:24,371 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:24,931 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:25,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:25,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:26,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:26,931 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:27,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:28,372 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:28,932 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:29,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:29,336 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:29,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:30,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:30,483 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:31,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:32,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:33,375 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:34,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:35,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:36,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:36,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:37,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:38,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:39,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:40,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:41,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:42,351 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:42,376 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:43,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:43,980 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:57:44,336 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:44,337 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:44,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:45,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:46,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:47,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:47,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:48,377 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:49,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:50,378 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:51,381 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:52,143 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:57:52,143 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 with policy now +2025-01-06 20:57:52,150 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:57:52,150 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:57:52,150 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:57:52,151 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:57:52,379 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:52,395 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/35co50qv-media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:52,940 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:57:53,166 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:53,242 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/vq8kfyfq-media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:57:53,387 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:54,385 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:55,392 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:56,395 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:56,941 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:57:57,395 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:58,403 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:57:58,760 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:57:59,353 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:57:59,354 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:57:59,397 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:00,396 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:00,943 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:01,399 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:02,399 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:03,400 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:04,361 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:04,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:05,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:06,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:07,407 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:08,401 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:09,407 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:10,357 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:10,405 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:11,410 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:12,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:13,414 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:13,981 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:58:14,344 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:14,345 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:14,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:15,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:15,489 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:16,412 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:16,708 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:16,708 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:16,708 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:16,948 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:16,948 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:17,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:18,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:18,949 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:19,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:20,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:20,949 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:21,358 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:21,415 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:22,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:22,950 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:23,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:24,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:25,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:26,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:27,355 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:27,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:28,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:29,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:29,340 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:29,416 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:30,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:31,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:32,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:32,702 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:32,953 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:33,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:34,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:34,954 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:35,417 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:36,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:37,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:37,706 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:38,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:38,956 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:39,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:40,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:41,418 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:42,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:43,356 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:43,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:43,981 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:58:44,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:44,340 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:44,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:44,958 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:45,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:46,419 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:47,425 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:48,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:48,445 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:48,636 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:48,636 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 with policy now +2025-01-06 20:58:48,640 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:48,640 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:48,640 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:48,640 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:48,878 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/qyd8pmgm-media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:48,959 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:48,959 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:48,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:48,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval +2025-01-06 20:58:49,275 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/9auudd0t-media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:58:49,425 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:50,438 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:50,960 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:51,433 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:52,435 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:53,436 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:53,517 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:54,427 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:54,961 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:55,432 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:56,438 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:56,962 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:57,449 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:57,754 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:57,754 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:57,755 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:57,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:58:57,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,177 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:58:58,178 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:58:58,208 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/7yp6xd7m-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,370 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:58,436 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:58,675 INFO wandb-upload_0:2116885 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgwipaqn0 +2025-01-06 20:58:58,763 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:58:58,777 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:58:58,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval +2025-01-06 20:58:58,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table +2025-01-06 20:58:59,060 INFO SenderThread:2116885 [sender.py:send_request_log_artifact():1525] logged artifact run-hle5fmwc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM1NDE0OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1ODg3OQ==', 'latestArtifact': None}} +2025-01-06 20:58:59,060 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:59,060 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:58:59,060 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:58:59,060 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:59,061 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,061 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,061 DEBUG SenderThread:2116885 [sender.py:send():379] send: telemetry +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,062 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,062 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,062 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,063 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,063 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,063 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,063 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,064 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,064 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,064 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,065 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,065 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,065 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,065 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,066 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,066 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,066 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:58:59,239 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/acbwd561-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:59,260 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/2l6wdk20-media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:58:59,340 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:58:59,422 INFO wandb-upload_0:2116885 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpm6b7sljq +2025-01-06 20:58:59,437 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:58:59,478 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/fwkuobur-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:58:59,770 INFO SenderThread:2116885 [sender.py:send_request_log_artifact():1525] logged artifact run-hle5fmwc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM1NDE2MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1ODg4NA==', 'latestArtifact': None}} +2025-01-06 20:58:59,770 DEBUG SenderThread:2116885 [sender.py:send():379] send: files +2025-01-06 20:58:59,770 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json with policy now +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:58:59,771 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:58:59,771 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:58:59,963 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:00,032 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/gs46ew22-media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:59:00,105 DEBUG SenderThread:2116885 [sender.py:send():379] send: exit +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():588] handling runtime: 256 +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_exit():594] send defer +2025-01-06 20:59:00,106 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,106 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:59:00,106 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,106 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:59:00,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:59:00,107 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:59:00,107 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:59:00,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:59:00,107 INFO HandlerThread:2116885 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:59:00,107 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:59:00,107 DEBUG SystemMonitor:2116885 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:59:00,108 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:59:00,108 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:59:00,160 INFO HandlerThread:2116885 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:59:00,160 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,160 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:59:00,160 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:59:00,161 DEBUG SenderThread:2116885 [sender.py:send():379] send: stats +2025-01-06 20:59:00,161 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,161 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:59:00,161 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:00,161 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:59:00,161 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:59:00,161 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:00,161 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:59:00,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:00,964 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:04,966 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:59:06,761 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:06,762 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:59:06,762 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:59:06,762 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,763 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:59:06,763 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send():379] send: history +2025-01-06 20:59:06,763 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:59:06,764 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,765 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:59:06,765 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:06,765 DEBUG SenderThread:2116885 [sender.py:send():379] send: summary +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:59:06,766 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,766 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:59:06,766 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:59:06,766 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:59:06,766 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:06,766 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:59:06,767 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:06,767 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:59:06,966 INFO Thread-12 :2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:07,107 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:08,465 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:59:08,465 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:08,465 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,465 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:59:08,466 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,466 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:59:08,466 INFO SenderThread:2116885 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:59:08,468 INFO SenderThread:2116885 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:59:08,470 INFO SenderThread:2116885 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:59:08,484 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:59:08,485 DEBUG SenderThread:2116885 [sender.py:send():379] send: artifact +2025-01-06 20:59:08,488 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,488 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:59:08,690 INFO SenderThread:2116885 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:59:08,691 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,691 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:59:08,691 INFO SenderThread:2116885 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml config.yaml +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json wandb-summary.json +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:59:08,968 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt requirements.txt +2025-01-06 20:59:08,970 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:59:08,970 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log output.log +2025-01-06 20:59:08,972 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/diff.patch diff.patch +2025-01-06 20:59:08,973 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:59:08,977 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 media/videos/eval/1_eval_1_f5a9fd56a60dca233f45.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 media/videos/eval/2_eval_2_482df1c0b75a61d8a84c.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 media/videos/eval/3_eval_3_697753ebc9b72b44c236.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 media/videos/eval/4_eval_4_26772b2cd069309bc708.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 media/videos/eval/0_eval_0_512fdb13a887694fdda3.mp4 +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json media/table/eval/total_success_rate_ci_6_fc242770b6dc78d2cc17.table.json +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:59:08,978 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:59:08,978 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:08,978 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:59:08,979 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:08,979 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:59:08,979 INFO SenderThread:2116885 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:59:09,108 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,108 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,148 INFO wandb-upload_2:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/config.yaml +2025-01-06 20:59:09,163 INFO wandb-upload_0:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/wandb-summary.json +2025-01-06 20:59:09,210 INFO wandb-upload_3:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/requirements.txt +2025-01-06 20:59:09,236 INFO wandb-upload_4:2116885 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/files/output.log +2025-01-06 20:59:09,247 INFO wandb-upload_1:2116885 [upload_job.py:push():130] Uploaded file /tmp/tmpk2asnlynwandb/hzghrb2g-events.out.tfevents.1736214884.leonmkim-ROG-Strix-G15CS-G15CS.2116716.0 +2025-01-06 20:59:09,448 INFO Thread-11 (_thread_body):2116885 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:59:09,448 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,448 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:59:09,448 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,448 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:59:09,448 INFO SenderThread:2116885 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:59:09,450 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,450 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:59:09,450 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:59:09,450 INFO SenderThread:2116885 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:59:09,648 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,648 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:59:09,648 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:59:09,648 INFO SenderThread:2116885 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:59:09,648 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:59:09,649 INFO HandlerThread:2116885 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send():379] send: final +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send():379] send: footer +2025-01-06 20:59:09,649 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: defer +2025-01-06 20:59:09,649 INFO SenderThread:2116885 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:59:09,649 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,650 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:59:09,650 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:59:09,651 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:59:09,652 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:59:09,653 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:59:09,712 DEBUG SenderThread:2116885 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:59:09,712 INFO MainThread:2116885 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:59:09,713 INFO MainThread:2116885 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:59:09,713 INFO MainThread:2116885 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:59:09,713 DEBUG HandlerThread:2116885 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:59:09,713 INFO HandlerThread:2116885 [handler.py:finish():882] shutting down handler +2025-01-06 20:59:10,651 INFO WriterThread:2116885 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [sender.py:finish():1615] shutting down sender +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:59:10,712 INFO SenderThread:2116885 [file_pusher.py:join():175] waiting for file pusher diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug.log b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e5f100751909aab93f92a101c2fe2bd6b56e768a --- /dev/null +++ b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Configure stats pid to 2116716 +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/settings +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:54:43,556 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug.log +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/logs/debug-internal.log +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc'} +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():619] starting backend +2025-01-06 20:54:43,557 INFO MainThread:2116716 [wandb_init.py:init():623] setting up manager +2025-01-06 20:54:43,561 INFO MainThread:2116716 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:54:43,561 INFO MainThread:2116716 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:54:43,579 INFO MainThread:2116716 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:54:43,585 INFO MainThread:2116716 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:54:43,898 INFO MainThread:2116716 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:54:43,964 INFO MainThread:2116716 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:54:43,965 INFO MainThread:2116716 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:54:44,334 INFO MainThread:2116716 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:54:44,335 INFO MainThread:2116716 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:54:44,337 INFO MainThread:2116716 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:54:44,338 INFO MainThread:2116716 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/hle5fmwc/tb, True +2025-01-06 20:55:01,442 INFO MainThread:2116716 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:59:10,714 WARNING MsgRouterThr:2116716 [router.py:message_loop():77] message_loop has been closed diff --git a/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b394c4b467a762182e32f836d6b5ef3e20207953 Binary files /dev/null and b/hle5fmwc/wandb/run-20250106_205443-hle5fmwc/run-hle5fmwc.wandb differ